From 6e4f52f8a2e510273149acbaf629521d1b4aec2e Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 16 Oct 2019 16:16:39 +0200 Subject: Introduce new ingestion pipeline structure, implement internal Likes with it. --- lib/pleroma/web/activity_pub/activity_pub.ex | 35 +++++++++++++ lib/pleroma/web/activity_pub/builder.ex | 43 ++++++++++++++++ lib/pleroma/web/activity_pub/object_validator.ex | 57 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 28 +++++++++++ lib/pleroma/web/common_api/common_api.ex | 29 ++++++++--- .../mastodon_api/controllers/status_controller.ex | 6 +-- 6 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/builder.ex create mode 100644 lib/pleroma/web/activity_pub/object_validator.ex create mode 100644 lib/pleroma/web/activity_pub/side_effects.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 364452b5d..f4fc45926 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -18,6 +18,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.SideEffects alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -123,6 +125,38 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop + @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()} + def common_pipeline(object, meta) do + with {_, {:ok, validated_object, meta}} <- + {:validate_object, ObjectValidator.validate(object, meta)}, + {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:persist_object, persist(mrfd_object, meta)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:execute_side_effects, SideEffects.handle(activity, meta)} do + {:ok, activity, meta} + else + e -> {:error, e} + end + end + + # TODO rewrite in with style + @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} + def persist(object, meta) do + local = Keyword.get(meta, :local) + {recipients, _, _} = get_recipients(object) + + {:ok, activity} = + Repo.insert(%Activity{ + data: object, + local: local, + recipients: recipients, + actor: object["actor"] + }) + + {:ok, activity, meta} + end + def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), @@ -130,6 +164,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), + # ??? {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, :ok <- Containment.contain_child(map), {:ok, map, object} <- insert_full_object(map) do diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex new file mode 100644 index 000000000..1787f1510 --- /dev/null +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -0,0 +1,43 @@ +defmodule Pleroma.Web.ActivityPub.Builder do + @moduledoc """ + This module builds the objects. Meant to be used for creating local objects. + + This module encodes our addressing policies and general shape of our objects. + """ + + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.User + alias Pleroma.Object + + @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()} + def like(actor, object) do + object_actor = User.get_cached_by_ap_id(object.data["actor"]) + + # Address the actor of the object, and our actor's follower collection if the post is public. + to = + if Visibility.is_public?(object) do + [actor.follower_address, object.data["actor"]] + else + [object.data["actor"]] + end + + # CC everyone who's been addressed in the object, except ourself and the object actor's + # follower collection + cc = + (object.data["to"] ++ (object.data["cc"] || [])) + |> List.delete(actor.ap_id) + |> List.delete(object_actor.follower_address) + + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "actor" => actor.ap_id, + "type" => "Like", + "object" => object.data["id"], + "to" => to, + "cc" => cc, + "context" => object.data["context"] + }, []} + end +end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex new file mode 100644 index 000000000..8ecad0dec --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidator do + @moduledoc """ + This module is responsible for validating an object (which can be an activity) + and checking if it is both well formed and also compatible with our view of + the system. + """ + + alias Pleroma.User + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Utils + + def validate_id(object, meta) do + with {_, true} <- {:id_presence, Map.has_key?(object, "id")} do + {:ok, object, meta} + else + e -> {:error, e} + end + end + + def validate_actor(object, meta) do + with {_, %User{}} <- {:actor_validation, User.get_cached_by_ap_id(object["actor"])} do + {:ok, object, meta} + else + e -> {:error, e} + end + end + + def common_validations(object, meta) do + with {_, {:ok, object, meta}} <- {:validate_id, validate_id(object, meta)}, + {_, {:ok, object, meta}} <- {:validate_actor, validate_actor(object, meta)} do + {:ok, object, meta} + else + e -> {:error, e} + end + end + + @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} + def validate(object, meta) + + def validate(%{"type" => "Like"} = object, meta) do + with {:ok, object, meta} <- common_validations(object, meta), + {_, %Object{} = liked_object} <- {:find_liked_object, Object.normalize(object["object"])}, + {_, nil} <- {:existing_like, Utils.get_existing_like(object["actor"], liked_object)} do + {:ok, object, meta} + else + e -> {:error, e} + end + end + + def validate(object, meta) do + common_validations(object, meta) + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex new file mode 100644 index 000000000..6d3e77a62 --- /dev/null +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -0,0 +1,28 @@ +defmodule Pleroma.Web.ActivityPub.SideEffects do + @moduledoc """ + This module looks at an inserted object and executes the side effects that it + implies. For example, a `Like` activity will increase the like count on the + liked object, a `Follow` activity will add the user to the follower + collection, and so on. + """ + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Object + alias Pleroma.Notification + + def handle(object, meta \\ []) + + # Tasks this handles: + # - Add like to object + # - Set up notification + def handle(%{data: %{"type" => "Like"}} = object, meta) do + liked_object = Object.get_by_ap_id(object.data["object"]) + Utils.add_like_to_object(object, liked_object) + Notification.create_notifications(object) + {:ok, object, meta} + end + + # Nothing to do + def handle(object, meta) do + {:ok, object, meta} + end +end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 386408d51..466beb724 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -17,6 +18,7 @@ defmodule Pleroma.Web.CommonAPI do import Pleroma.Web.CommonAPI.Utils require Pleroma.Constants + require Logger def follow(follower, followed) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) @@ -98,16 +100,31 @@ defmodule Pleroma.Web.CommonAPI do end end - def favorite(id_or_ap_id, user) do - with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity), - nil <- Utils.get_existing_like(user.ap_id, object) do - ActivityPub.like(user, object) + @spec favorite(User.t(), binary()) :: {:ok, Activity.t()} | {:error, any()} + def favorite(%User{} = user, id) do + with {_, %Activity{object: object}} <- {:find_object, Activity.get_by_id_with_object(id)}, + {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, + {_, {:ok, %Activity{} = activity, _meta}} <- + {:common_pipeline, + ActivityPub.common_pipeline(like_object, Keyword.put(meta, :local, true))} do + {:ok, activity} else - _ -> {:error, dgettext("errors", "Could not favorite")} + e -> + Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}") + {:error, dgettext("errors", "Could not favorite")} end end + # def favorite(id_or_ap_id, user) do + # with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), + # object <- Object.normalize(activity), + # nil <- Utils.get_existing_like(user.ap_id, object) do + # ActivityPub.like(user, object) + # else + # _ -> {:error, dgettext("errors", "Could not favorite")} + # end + # end + def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id) do object = Object.normalize(activity) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index e5d016f63..4b4482aa8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -201,9 +201,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "POST /api/v1/statuses/:id/favourite" - def favourite(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do - with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do + def favourite(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) do + with {:ok, _fav} <- CommonAPI.favorite(user, activity_id), + %Activity{} = activity <- Activity.get_by_id(activity_id) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end -- cgit v1.2.3 From 081e8206ab75e336a76b621508b3999170159ec6 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 16 Oct 2019 17:03:21 +0200 Subject: Transmogrifier: Use new ingestion pipeline for Likes. --- lib/pleroma/object/containment.ex | 12 +++++++++ lib/pleroma/web/activity_pub/object_validator.ex | 5 ++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 31 +++++++++++++++++++----- 3 files changed, 40 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index f077a9f32..edbe92381 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -32,6 +32,18 @@ defmodule Pleroma.Object.Containment do get_actor(%{"actor" => actor}) end + def get_object(%{"object" => id}) when is_binary(id) do + id + end + + def get_object(%{"object" => %{"id" => id}}) when is_binary(id) do + id + end + + def get_object(_) do + nil + end + @doc """ Checks that an imported AP object's actor matches the domain it came from. """ diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 8ecad0dec..0048cc4ec 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def common_validations(object, meta) do with {_, {:ok, object, meta}} <- {:validate_id, validate_id(object, meta)}, - {_, {:ok, object, meta}} <- {:validate_actor, validate_actor(object, meta)} do + {_, {:ok, object, meta}} <- {:validate_actor, validate_actor(object, meta)} do {:ok, object, meta} else e -> {:error, e} @@ -43,7 +43,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => "Like"} = object, meta) do with {:ok, object, meta} <- common_validations(object, meta), - {_, %Object{} = liked_object} <- {:find_liked_object, Object.normalize(object["object"])}, + {_, %Object{} = liked_object} <- + {:find_liked_object, Object.normalize(object["object"])}, {_, nil} <- {:existing_like, Utils.get_existing_like(object["actor"], liked_object)} do {:ok, object, meta} else diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b56343beb..3e982adcb 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -563,19 +563,38 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + %{"type" => "Like", "object" => _object_id, "actor" => _actor, "id" => _id} = data, _options ) do - with actor <- Containment.get_actor(data), - {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id), - {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do + with data <- Map.take(data, ["type", "object", "actor", "context", "id"]), + actor <- Containment.get_actor(data), + object <- Containment.get_object(data), + data <- data |> Map.put("actor", actor) |> Map.put("object", object), + _user <- User.get_or_fetch_by_ap_id(actor), + object <- Object.normalize(object), + data <- Map.put_new(data, "context", object.data["context"]), + {_, {:ok, activity, _meta}} <- + {:common_pipeline, ActivityPub.common_pipeline(data, local: false)} do {:ok, activity} else - _e -> :error + e -> {:error, e} end end + # def handle_incoming( + # %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + # _options + # ) do + # with actor <- Containment.get_actor(data), + # {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), + # {:ok, object} <- get_obj_helper(object_id), + # {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do + # {:ok, activity} + # else + # _e -> :error + # end + # end + def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, _options -- cgit v1.2.3 From 66452f518faa1f079f02006943b0c2cdc830b47f Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 17 Oct 2019 18:36:52 +0200 Subject: ObjectValidator: Rewrite LikeValidator with Ecto. --- lib/pleroma/web/activity_pub/object_validator.ex | 42 +++---------- .../object_validators/like_validator.ex | 69 ++++++++++++++++++++++ .../activity_pub/object_validators/types/object.ex | 25 ++++++++ lib/pleroma/web/common_api/common_api.ex | 10 ---- 4 files changed, 102 insertions(+), 44 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/like_validator.ex create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/object.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 0048cc4ec..adcb53c65 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -9,50 +9,24 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do the system. """ - alias Pleroma.User - alias Pleroma.Object - alias Pleroma.Web.ActivityPub.Utils - - def validate_id(object, meta) do - with {_, true} <- {:id_presence, Map.has_key?(object, "id")} do - {:ok, object, meta} - else - e -> {:error, e} - end - end - - def validate_actor(object, meta) do - with {_, %User{}} <- {:actor_validation, User.get_cached_by_ap_id(object["actor"])} do - {:ok, object, meta} - else - e -> {:error, e} - end - end - - def common_validations(object, meta) do - with {_, {:ok, object, meta}} <- {:validate_id, validate_id(object, meta)}, - {_, {:ok, object, meta}} <- {:validate_actor, validate_actor(object, meta)} do - {:ok, object, meta} - else - e -> {:error, e} - end - end + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) def validate(%{"type" => "Like"} = object, meta) do - with {:ok, object, meta} <- common_validations(object, meta), - {_, %Object{} = liked_object} <- - {:find_liked_object, Object.normalize(object["object"])}, - {_, nil} <- {:existing_like, Utils.get_existing_like(object["actor"], liked_object)} do + with {_, %{valid?: true, changes: object}} <- + {:validate_object, LikeValidator.cast_and_validate(object)} do + object = stringify_keys(object) {:ok, object, meta} else e -> {:error, e} end end - def validate(object, meta) do - common_validations(object, meta) + defp stringify_keys(object) do + object + |> Enum.map(fn {key, val} -> {to_string(key), val} end) + |> Enum.into(%{}) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex new file mode 100644 index 000000000..d5a2f7202 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -0,0 +1,69 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do + use Ecto.Schema + import Ecto.Changeset + + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.User + alias Pleroma.Object + + @primary_key false + + embedded_schema do + field(:id, :string, primary_key: true) + field(:type, :string) + field(:object, Types.ObjectID) + field(:actor, Types.ObjectID) + field(:context, :string) + field(:to, {:array, :string}) + field(:cc, {:array, :string}) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, [:id, :type, :object, :actor, :context, :to, :cc]) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Like"]) + |> validate_required([:id, :type, :object, :actor, :context]) + |> validate_change(:actor, &actor_valid?/2) + |> validate_change(:object, &object_valid?/2) + |> validate_existing_like() + end + + def validate_existing_like(%{changes: %{actor: actor, object: object}} = cng) do + if Utils.get_existing_like(actor, %{data: %{"id" => object}}) do + cng + |> add_error(:actor, "already liked this object") + |> add_error(:object, "already liked by this actor") + else + cng + end + end + + def validate_existing_like(cng), do: cng + + def actor_valid?(field_name, actor) do + if User.get_cached_by_ap_id(actor) do + [] + else + [{field_name, "can't find user"}] + end + end + + def object_valid?(field_name, object) do + if Object.get_cached_by_ap_id(object) do + [] + else + [{field_name, "can't find object"}] + end + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object.ex b/lib/pleroma/web/activity_pub/object_validators/types/object.ex new file mode 100644 index 000000000..92fc13ba8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/object.ex @@ -0,0 +1,25 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do + use Ecto.Type + + def type, do: :string + + def cast(object) when is_binary(object) do + {:ok, object} + end + + def cast(%{"id" => object}) when is_binary(object) do + {:ok, object} + end + + def cast(_) do + :error + end + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 466beb724..e0b22a314 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -115,16 +115,6 @@ defmodule Pleroma.Web.CommonAPI do end end - # def favorite(id_or_ap_id, user) do - # with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - # object <- Object.normalize(activity), - # nil <- Utils.get_existing_like(user.ap_id, object) do - # ActivityPub.like(user, object) - # else - # _ -> {:error, dgettext("errors", "Could not favorite")} - # end - # end - def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id) do object = Object.normalize(activity) -- cgit v1.2.3 From 203d61b95012fd2cb8a8618f6f51a3748c940cc1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 17 Oct 2019 19:35:31 +0200 Subject: Transmogrifier: Make proper use of the LikeValidator. --- lib/pleroma/web/activity_pub/object_validator.ex | 10 ++- .../object_validators/like_validator.ex | 2 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 79 +++++++++++++++------- 3 files changed, 63 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index adcb53c65..33e67dbb9 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do """ alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.User + alias Pleroma.Object @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) @@ -24,9 +26,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - defp stringify_keys(object) do + def stringify_keys(object) do object |> Enum.map(fn {key, val} -> {to_string(key), val} end) |> Enum.into(%{}) end + + def fetch_actor_and_object(object) do + User.get_or_fetch_by_ap_id(object["actor"]) + Object.normalize(object["object"]) + :ok + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index d5a2f7202..e6a5aaca8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -33,7 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do def validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Like"]) - |> validate_required([:id, :type, :object, :actor, :context]) + |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) |> validate_change(:actor, &actor_valid?/2) |> validate_change(:object, &object_valid?/2) |> validate_existing_like() diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3e982adcb..591d7aa94 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -16,6 +16,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator alias Pleroma.Workers.TransmogrifierWorker + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator import Ecto.Query @@ -562,39 +564,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming( - %{"type" => "Like", "object" => _object_id, "actor" => _actor, "id" => _id} = data, - _options - ) do - with data <- Map.take(data, ["type", "object", "actor", "context", "id"]), - actor <- Containment.get_actor(data), - object <- Containment.get_object(data), - data <- data |> Map.put("actor", actor) |> Map.put("object", object), - _user <- User.get_or_fetch_by_ap_id(actor), - object <- Object.normalize(object), - data <- Map.put_new(data, "context", object.data["context"]), + def handle_incoming(%{"type" => "Like"} = data, _options) do + with {_, %{changes: cast_data}} <- {:casting_data, LikeValidator.cast_data(data)}, + cast_data <- ObjectValidator.stringify_keys(cast_data), + :ok <- ObjectValidator.fetch_actor_and_object(cast_data), + {_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)}, + {_, {:ok, cast_data}} <- + {:maybe_add_recipients, maybe_add_recipients_from_object(cast_data)}, {_, {:ok, activity, _meta}} <- - {:common_pipeline, ActivityPub.common_pipeline(data, local: false)} do + {:common_pipeline, ActivityPub.common_pipeline(cast_data, local: false)} do {:ok, activity} else e -> {:error, e} end end - # def handle_incoming( - # %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, - # _options - # ) do - # with actor <- Containment.get_actor(data), - # {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), - # {:ok, object} <- get_obj_helper(object_id), - # {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do - # {:ok, activity} - # else - # _e -> :error - # end - # end - def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, _options @@ -1156,4 +1140,47 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_user_url(data), do: data def maybe_fix_user_object(data), do: maybe_fix_user_url(data) + + defp maybe_add_context_from_object(%{"context" => context} = data) when is_binary(context), + do: {:ok, data} + + defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(object) do + if object = Object.normalize(object) do + data = + data + |> Map.put("context", object.data["context"]) + + {:ok, data} + else + {:error, "No context on referenced object"} + end + end + + defp maybe_add_context_from_object(_) do + {:error, "No referenced object"} + end + + defp maybe_add_recipients_from_object(%{"object" => object} = data) do + to = data["to"] || [] + cc = data["cc"] || [] + + if to == [] && cc == [] do + if object = Object.normalize(object) do + data = + data + |> Map.put("to", [object.data["actor"]]) + |> Map.put("cc", cc) + + {:ok, data} + else + {:error, "No actor on referenced object"} + end + else + {:ok, data} + end + end + + defp maybe_add_recipients_from_object(_) do + {:error, "No referenced object"} + end end -- cgit v1.2.3 From 97d5c79aa07bfe836cd676424ce1b5a298c72b60 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 23 Oct 2019 11:52:27 +0200 Subject: Add Pipeline module, test for federation. --- lib/pleroma/web/activity_pub/activity_pub.ex | 19 +----------- lib/pleroma/web/activity_pub/pipeline.ex | 41 ++++++++++++++++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 7 +++-- lib/pleroma/web/common_api/common_api.ex | 3 +- 4 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/pipeline.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f4fc45926..0789ec31c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -18,8 +18,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.ActivityPub.ObjectValidator - alias Pleroma.Web.ActivityPub.SideEffects alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -125,25 +123,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop - @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()} - def common_pipeline(object, meta) do - with {_, {:ok, validated_object, meta}} <- - {:validate_object, ObjectValidator.validate(object, meta)}, - {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)}, - {_, {:ok, %Activity{} = activity, meta}} <- - {:persist_object, persist(mrfd_object, meta)}, - {_, {:ok, %Activity{} = activity, meta}} <- - {:execute_side_effects, SideEffects.handle(activity, meta)} do - {:ok, activity, meta} - else - e -> {:error, e} - end - end - # TODO rewrite in with style @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(object, meta) do - local = Keyword.get(meta, :local) + local = Keyword.fetch!(meta, :local) {recipients, _, _} = get_recipients(object) {:ok, activity} = diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex new file mode 100644 index 000000000..cb3571917 --- /dev/null +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Pipeline do + alias Pleroma.Activity + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.MRF + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.SideEffects + alias Pleroma.Web.Federator + + @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()} + def common_pipeline(object, meta) do + with {_, {:ok, validated_object, meta}} <- + {:validate_object, ObjectValidator.validate(object, meta)}, + {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:persist_object, ActivityPub.persist(mrfd_object, meta)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:execute_side_effects, SideEffects.handle(activity, meta)}, + {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do + {:ok, activity, meta} + else + e -> {:error, e} + end + end + + defp maybe_federate(activity, meta) do + with {:ok, local} <- Keyword.fetch(meta, :local) do + if local do + Federator.publish(activity) + {:ok, :federated} + else + {:ok, :not_federated} + end + else + _e -> {:error, "local not set in meta"} + end + end +end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 591d7aa94..4dd884ce9 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -12,12 +12,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator alias Pleroma.Workers.TransmogrifierWorker - alias Pleroma.Web.ActivityPub.ObjectValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator import Ecto.Query @@ -572,7 +573,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {_, {:ok, cast_data}} <- {:maybe_add_recipients, maybe_add_recipients_from_object(cast_data)}, {_, {:ok, activity, _meta}} <- - {:common_pipeline, ActivityPub.common_pipeline(cast_data, local: false)} do + {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do {:ok, activity} else e -> {:error, e} diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e0b22a314..535a48dcc 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -106,7 +107,7 @@ defmodule Pleroma.Web.CommonAPI do {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, {_, {:ok, %Activity{} = activity, _meta}} <- {:common_pipeline, - ActivityPub.common_pipeline(like_object, Keyword.put(meta, :local, true))} do + Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do {:ok, activity} else e -> -- cgit v1.2.3 From 1adafa096653c4538e4162a2dffba982ee6c6d8e Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 23 Oct 2019 12:18:05 +0200 Subject: Credo fixes. --- lib/pleroma/web/activity_pub/builder.ex | 4 ++-- lib/pleroma/web/activity_pub/object_validator.ex | 4 ++-- lib/pleroma/web/activity_pub/object_validators/like_validator.ex | 8 ++++++-- lib/pleroma/web/activity_pub/side_effects.ex | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 1787f1510..429a510b8 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -5,10 +5,10 @@ defmodule Pleroma.Web.ActivityPub.Builder do This module encodes our addressing policies and general shape of our objects. """ + alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.User - alias Pleroma.Object @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()} def like(actor, object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 33e67dbb9..27a8dd852 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -9,9 +9,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do the system. """ - alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator - alias Pleroma.User alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index e6a5aaca8..5fa486653 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -1,11 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do use Ecto.Schema import Ecto.Changeset + alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.User - alias Pleroma.Object @primary_key false diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 6d3e77a62..666a4e310 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do liked object, a `Follow` activity will add the user to the follower collection, and so on. """ - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Object alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Utils def handle(object, meta \\ []) -- cgit v1.2.3 From 3d1b445cbf001f76af614441c241dcc299e76af7 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 5 Nov 2019 15:02:09 +0100 Subject: Object Validators: Extract common validations. --- lib/pleroma/web/activity_pub/object_validator.ex | 7 +++-- .../object_validators/common_validations.ex | 32 ++++++++++++++++++++++ .../object_validators/like_validator.ex | 26 ++++-------------- lib/pleroma/web/activity_pub/transmogrifier.ex | 7 +++-- 4 files changed, 46 insertions(+), 26 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/common_validations.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 27a8dd852..539be1143 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,9 +17,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(object, meta) def validate(%{"type" => "Like"} = object, meta) do - with {_, %{valid?: true, changes: object}} <- - {:validate_object, LikeValidator.cast_and_validate(object)} do - object = stringify_keys(object) + with {_, {:ok, object}} <- + {:validate_object, + object |> LikeValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert)} do + object = stringify_keys(object |> Map.from_struct()) {:ok, object, meta} else e -> {:error, e} diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex new file mode 100644 index 000000000..db0e2072d --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do + import Ecto.Changeset + + alias Pleroma.Object + alias Pleroma.User + + def validate_actor_presence(cng, field_name \\ :actor) do + cng + |> validate_change(field_name, fn field_name, actor -> + if User.get_cached_by_ap_id(actor) do + [] + else + [{field_name, "can't find user"}] + end + end) + end + + def validate_object_presence(cng, field_name \\ :object) do + cng + |> validate_change(field_name, fn field_name, actor -> + if Object.get_cached_by_ap_id(actor) do + [] + else + [{field_name, "can't find user"}] + end + end) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 5fa486653..ccbc7d071 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -4,13 +4,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do use Ecto.Schema - import Ecto.Changeset - alias Pleroma.Object - alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + @primary_key false embedded_schema do @@ -38,8 +38,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do data_cng |> validate_inclusion(:type, ["Like"]) |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) - |> validate_change(:actor, &actor_valid?/2) - |> validate_change(:object, &object_valid?/2) + |> validate_actor_presence() + |> validate_object_presence() |> validate_existing_like() end @@ -54,20 +54,4 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do end def validate_existing_like(cng), do: cng - - def actor_valid?(field_name, actor) do - if User.get_cached_by_ap_id(actor) do - [] - else - [{field_name, "can't find user"}] - end - end - - def object_valid?(field_name, object) do - if Object.get_cached_by_ap_id(object) do - [] - else - [{field_name, "can't find object"}] - end - end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4dd884ce9..9a0c37e13 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -566,8 +566,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => "Like"} = data, _options) do - with {_, %{changes: cast_data}} <- {:casting_data, LikeValidator.cast_data(data)}, - cast_data <- ObjectValidator.stringify_keys(cast_data), + with {_, {:ok, cast_data_sym}} <- + {:casting_data, + data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)}, + {_, cast_data} <- + {:stringify_keys, ObjectValidator.stringify_keys(cast_data_sym |> Map.from_struct())}, :ok <- ObjectValidator.fetch_actor_and_object(cast_data), {_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)}, {_, {:ok, cast_data}} <- -- cgit v1.2.3 From faced6236b9e2ce9675cf743068f16098b744562 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 5 Nov 2019 15:02:31 +0100 Subject: NoteValidator: Add very basic validator for Note objects. --- .../object_validators/note_validator.ex | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/object_validators/note_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex new file mode 100644 index 000000000..c660f30f0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:id, :string, primary_key: true) + field(:to, {:array, :string}, default: []) + field(:cc, {:array, :string}, default: []) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + field(:actor, Types.ObjectID) + field(:attributedTo, Types.ObjectID) + field(:summary, :string) + # TODO: Write type + field(:published, :string) + # TODO: Write type + field(:emoji, :map, default: %{}) + field(:sensitive, :boolean, default: false) + # TODO: Write type + field(:attachment, {:array, :map}, default: []) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inRepyTo, :string) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + + # see if needed + field(:conversation, :string) + field(:context_id, :string) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Note"]) + |> validate_required([:id, :actor, :to, :cc, :type, :content, :context]) + end +end -- cgit v1.2.3 From 1993d7096d673d8a8151fedd7bcac909d584d13d Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 5 Dec 2019 12:33:06 +0100 Subject: Validators: Add a type for the datetime used in AP. --- .../object_validators/note_validator.ex | 3 +- .../object_validators/types/date_time.ex | 34 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/date_time.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index c660f30f0..eea15ce1c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -25,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:actor, Types.ObjectID) field(:attributedTo, Types.ObjectID) field(:summary, :string) - # TODO: Write type - field(:published, :string) + field(:published, Types.DateTime) # TODO: Write type field(:emoji, :map, default: %{}) field(:sensitive, :boolean, default: false) diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex new file mode 100644 index 000000000..4f412fcde --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex @@ -0,0 +1,34 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do + @moduledoc """ + The AP standard defines the date fields in AP as xsd:DateTime. Elixir's + DateTime can't parse this, but it can parse the related iso8601. This + module punches the date until it looks like iso8601 and normalizes to + it. + + DateTimes without a timezone offset are treated as UTC. + + Reference: https://www.w3.org/TR/activitystreams-vocabulary/#dfn-published + """ + use Ecto.Type + + def type, do: :string + + def cast(datetime) when is_binary(datetime) do + with {:ok, datetime, _} <- DateTime.from_iso8601(datetime) do + {:ok, DateTime.to_iso8601(datetime)} + else + {:error, :missing_offset} -> cast("#{datetime}Z") + _e -> :error + end + end + + def cast(_), do: :error + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end -- cgit v1.2.3 From d4bafabfd14887e61eb5bc1d877035dcfebbd33f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 9 Dec 2019 10:39:14 +0100 Subject: Beginnings of the create validator --- .../object_validators/create_validator.ex | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/object_validators/create_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex new file mode 100644 index 000000000..bd90f7250 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:id, :string, primary_key: true) + field(:actor, Types.ObjectID) + field(:type, :string) + field(:to, {:array, :string}) + field(:cc, {:array, :string}) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + + embeds_one(:object, NoteValidator) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end +end -- cgit v1.2.3 From 514c899275a32e6ef63305f9424c50344d41b12e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 11 Feb 2020 10:12:57 +0300 Subject: adding gun adapter --- lib/mix/tasks/pleroma/benchmark.ex | 39 ++ lib/mix/tasks/pleroma/emoji.ex | 9 +- lib/pleroma/application.ex | 90 +++-- lib/pleroma/config/config_db.ex | 11 - lib/pleroma/config/transfer_task.ex | 43 ++- lib/pleroma/gun/api.ex | 26 ++ lib/pleroma/gun/api/mock.ex | 151 ++++++++ lib/pleroma/gun/conn.ex | 29 ++ lib/pleroma/gun/gun.ex | 45 +++ lib/pleroma/http/adapter.ex | 64 ++++ lib/pleroma/http/adapter/gun.ex | 123 ++++++ lib/pleroma/http/adapter/hackney.ex | 41 ++ lib/pleroma/http/connection.ex | 113 ++++-- lib/pleroma/http/http.ex | 154 +++++--- lib/pleroma/http/request.ex | 23 ++ lib/pleroma/http/request_builder.ex | 105 ++---- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/otp_version.ex | 63 ++++ lib/pleroma/pool/connections.ex | 415 +++++++++++++++++++++ lib/pleroma/pool/pool.ex | 22 ++ lib/pleroma/pool/request.ex | 72 ++++ lib/pleroma/pool/supervisor.ex | 36 ++ lib/pleroma/reverse_proxy/client.ex | 26 +- lib/pleroma/reverse_proxy/client/hackney.ex | 24 ++ lib/pleroma/reverse_proxy/client/tesla.ex | 87 +++++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 20 +- .../activity_pub/mrf/media_proxy_warming_policy.ex | 14 +- lib/pleroma/web/rel_me.ex | 18 +- lib/pleroma/web/rich_media/parser.ex | 18 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 30 files changed, 1647 insertions(+), 242 deletions(-) create mode 100644 lib/pleroma/gun/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/gun.ex create mode 100644 lib/pleroma/http/adapter.ex create mode 100644 lib/pleroma/http/adapter/gun.ex create mode 100644 lib/pleroma/http/adapter/hackney.ex create mode 100644 lib/pleroma/http/request.ex create mode 100644 lib/pleroma/otp_version.ex create mode 100644 lib/pleroma/pool/connections.ex create mode 100644 lib/pleroma/pool/pool.ex create mode 100644 lib/pleroma/pool/request.ex create mode 100644 lib/pleroma/pool/supervisor.ex create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 84dccf7f3..01e079136 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -74,4 +74,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do inputs: inputs ) end + + def run(["adapters"]) do + start_pleroma() + + :ok = + Pleroma.Pool.Connections.open_conn( + "https://httpbin.org/stream-bytes/1500", + :gun_connections + ) + + Process.sleep(1_500) + + Benchee.run( + %{ + "Without conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool, receive_conn: false] + ) + end, + "Without conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [receive_conn: false] + ) + end, + "With reused conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool] + ) + end, + "With reused conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") + end + }, + parallel: 10 + ) + end end diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 24d999707..b4e8d3a0b 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -4,13 +4,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do use Mix.Task + import Mix.Pleroma @shortdoc "Manages emoji packs" @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md") def run(["ls-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, [], []} = parse_global_opts(args) @@ -36,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do end def run(["get-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, pack_names, []} = parse_global_opts(args) @@ -135,7 +134,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do end def run(["gen-pack", src]) do - Application.ensure_all_started(:hackney) + start_pleroma() proposed_name = Path.basename(src) |> Path.rootname() name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 27758cf94..df6d3a98d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,8 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do - import Cachex.Spec use Application + + import Cachex.Spec + + alias Pleroma.Config + require Logger @name Mix.Project.config()[:name] @@ -18,9 +22,9 @@ defmodule Pleroma.Application do def repository, do: @repository def user_agent do - case Pleroma.Config.get([:http, :user_agent], :default) do + case Config.get([:http, :user_agent], :default) do :default -> - info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" named_version() <> "; " <> info custom -> @@ -32,7 +36,7 @@ defmodule Pleroma.Application do # for more information on OTP Applications def start(_type, _args) do Pleroma.HTML.compile_scrubbers() - Pleroma.Config.DeprecationWarnings.warn() + Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Repo.check_migrations_applied!() setup_instrumenters() @@ -42,17 +46,17 @@ defmodule Pleroma.Application do children = [ Pleroma.Repo, - Pleroma.Config.TransferTask, + Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - hackney_pool_children() ++ + http_pools_children(Config.get(:env)) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, Pleroma.Config.get(Oban)} + {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -62,6 +66,18 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] + case Pleroma.OTPVersion.check_version() do + :ok -> :ok + {:error, version} -> raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + :undefined -> raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] @@ -69,7 +85,7 @@ defmodule Pleroma.Application do end def load_custom_modules do - dir = Pleroma.Config.get([:modules, :runtime_dir]) + dir = Config.get([:modules, :runtime_dir]) if dir && File.exists?(dir) do dir @@ -110,20 +126,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -145,7 +147,7 @@ defmodule Pleroma.Application do do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), do: %{ @@ -154,7 +156,7 @@ defmodule Pleroma.Application do type: :worker } - defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp chat_enabled?, do: Config.get([:chat, :enabled]) defp streamer_child(:test), do: [] @@ -168,13 +170,6 @@ defmodule Pleroma.Application do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) - end - end - defp task_children(:test) do [ %{ @@ -199,4 +194,37 @@ defmodule Pleroma.Application do } ] end + + # start hackney and gun pools in tests + defp http_pools_children(:test) do + hackney_options = Config.get([:hackney_pools, :federation]) + hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) + [hackney_pool, Pleroma.Pool.Supervisor] + end + + defp http_pools_children(_) do + :tesla + |> Application.get_env(:adapter) + |> http_pools() + end + + defp http_pools(Tesla.Adapter.Hackney) do + pools = [:federation, :media] + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) do + [:upload | pools] + else + pools + end + + for pool <- pools do + options = Config.get([:hackney_pools, pool]) + :hackney_pool.child_spec(pool, options) + end + end + + defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] + + defp http_pools(_), do: [] end diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 119251bee..bdacefa97 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -278,8 +278,6 @@ defmodule Pleroma.ConfigDB do } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do value = entity @@ -323,15 +321,6 @@ defmodule Pleroma.ConfigDB do {:proxy_url, {do_transform_string(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = - entity - |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") - |> Code.eval_string() - - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6c5ba1f95..251074aaa 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -18,7 +18,10 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Oban}, {:pleroma, :rate_limit}, {:pleroma, :markup}, - {:plerome, :streamer} + {:pleroma, :streamer}, + {:pleroma, :pools}, + {:pleroma, :connections_pool}, + {:tesla, :adapter} ] @reboot_time_subkeys [ @@ -74,6 +77,28 @@ defmodule Pleroma.Config.TransferTask do end end + defp group_for_restart(:logger, key, _, merged_value) do + # change logger configuration in runtime, without restart + if Keyword.keyword?(merged_value) and + key not in [:compile_time_application, :backends, :compile_time_purge_matching] do + Logger.configure_backend(key, merged_value) + else + Logger.configure([{key, merged_value}]) + end + + nil + end + + defp group_for_restart(:tesla, _, _, _), do: :pleroma + + defp group_for_restart(group, _, _, _) when group != :pleroma, do: group + + defp group_for_restart(group, key, value, _) do + if pleroma_need_restart?(group, key, value) do + group + end + end + defp merge_and_update(setting) do try do key = ConfigDB.from_string(setting.key) @@ -95,21 +120,7 @@ defmodule Pleroma.Config.TransferTask do :ok = update_env(group, key, merged_value) - if group != :logger do - if group != :pleroma or pleroma_need_restart?(group, key, value) do - group - end - else - # change logger configuration in runtime, without restart - if Keyword.keyword?(merged_value) and - key not in [:compile_time_application, :backends, :compile_time_purge_matching] do - Logger.configure_backend(key, merged_value) - else - Logger.configure([{key, merged_value}]) - end - - nil - end + group_for_restart(group, key, value, merged_value) rescue error -> error_msg = diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex new file mode 100644 index 000000000..a0c3c5415 --- /dev/null +++ b/lib/pleroma/gun/api.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + + def open(host, port, opts), do: api().open(host, port, opts) + + def info(pid), do: api().info(pid) + + def close(pid), do: api().close(pid) + + def await_up(pid), do: api().await_up(pid) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..0134b016e --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,151 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API + def open('some-domain.com', 443, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(ip, port, _) + when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and + port in [80, 443] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + scheme = if port == 443, do: "https", else: "http" + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: scheme, + origin_host: ip, + origin_port: port + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks5, 'localhost', 1234}, + socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'proxy-socks.com', + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open('localhost', 1234, %{ + protocols: [:socks], + proxy: {:socks4, 'localhost', 1234}, + socks_opts: %{ + host: 'proxy-socks.com', + port: 443, + protocols: [:http2], + tls_opts: [], + transport: :tls, + version: 4 + } + }) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'proxy-socks.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} + + @impl API + def open('example.com', port, _) when port in [443, 115] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'example.com', + origin_port: 443 + }) + + {:ok, conn_pid} + end + + @impl API + def open(domain, 80, _) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + + {:ok, conn_pid} + end + + @impl API + def open({127, 0, 0, 1}, 8123, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def open('localhost', 9050, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def await_up(_pid), do: {:ok, :http} + + @impl API + def connect(pid, %{host: _, port: 80}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def await(pid, ref) do + [{_, ^pid}] = Registry.lookup(API.Mock, ref) + {:response, :fin, 200, []} + end + + @impl API + def info(pid) do + [{_, info}] = Registry.lookup(API.Mock, pid) + info + end + + @impl API + def close(_pid), do: :ok +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..2474829d6 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type gun_state :: :up | :down + @type conn_state :: :active | :idle + + @type t :: %__MODULE__{ + conn: pid(), + gun_state: gun_state(), + conn_state: conn_state(), + used_by: [pid()], + last_reference: pos_integer(), + crf: float(), + retries: pos_integer() + } + + defstruct conn: nil, + gun_state: :open, + conn_state: :init, + used_by: [], + last_reference: 0, + crf: 1, + retries: 0 +end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex new file mode 100644 index 000000000..4a1bbc95f --- /dev/null +++ b/lib/pleroma/gun/gun.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] + + @impl API + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + + @impl API + defdelegate info(pid), to: :gun + + @impl API + defdelegate close(pid), to: :gun + + @impl API + defdelegate await_up(pid), to: :gun + + @impl API + defdelegate connect(pid, opts), to: :gun + + @impl API + defdelegate await(pid, ref), to: :gun + + @spec flush(pid() | reference()) :: :ok + defdelegate flush(pid), to: :gun +end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex new file mode 100644 index 000000000..6166a3eb4 --- /dev/null +++ b/lib/pleroma/http/adapter.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), pos_integer()} + @type host_type :: :domain | :ip + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do + {host, port} + else + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @spec domain_or_fallback(String.t()) :: charlist() + def domain_or_fallback(host) do + case domain_or_ip(host) do + {:domain, domain} -> domain + {:ip, _ip} -> to_charlist(host) + end + end + + @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} + def domain_or_ip(host) do + charlist = to_charlist(host) + + case :inet.parse_address(charlist) do + {:error, :einval} -> + {:domain, :idna.encode(charlist)} + + {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ip, ip} + end + end +end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex new file mode 100644 index 000000000..f25afeda7 --- /dev/null +++ b/lib/pleroma/http/adapter/gun.ex @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.Gun do + @behaviour Pleroma.HTTP.Adapter + + alias Pleroma.HTTP.Adapter + + require Logger + + alias Pleroma.Pool.Connections + + @defaults [ + connect_timeout: 20_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry_timeout: 100, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> add_original(uri) + |> add_scheme_opts(uri) + |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) + |> maybe_get_conn(uri, connection_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + with conn when not is_nil(conn) <- opts[:conn], + body_as when body_as != :chunks <- opts[:body_as] do + Connections.checkout(conn, self(), :gun_connections) + end + + :ok + end + + defp add_original(opts, %URI{host: host, port: port}) do + formatted_host = Adapter.domain_or_fallback(host) + + Keyword.put(opts, :original, "#{formatted_host}:#{port}") + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + adapter_opts = [ + certificates_verification: true, + tls_opts: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + ] + ] + + adapter_opts = + if port != 443 do + Keyword.put(adapter_opts, :transport, :tls) + else + adapter_opts + end + + Keyword.merge(opts, adapter_opts) + end + + defp maybe_get_conn(adapter_opts, uri, connection_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(connection_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + try_to_get_conn(uri, opts) + else + opts + end + end + + defp try_to_get_conn(uri, opts) do + try do + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.info( + "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + ) + + :ok = Connections.open_conn(uri, :gun_connections, opts) + opts + + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + + opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) + end + rescue + error -> + Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + opts + catch + :exit, {:timeout, _} -> + Logger.info( + "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + ) + + opts + + :exit, error -> + Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + opts + end + end +end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter/hackney.ex new file mode 100644 index 000000000..00db30083 --- /dev/null +++ b/lib/pleroma/http/adapter/hackney.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.HTTP.Adapter.Hackney do + @behaviour Pleroma.HTTP.Adapter + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..85918341a 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -4,40 +4,99 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ - Connection for http-requests. + Configure Tesla.Client with default and customized adapter options. """ + @type ip_address :: ipv4_address() | ipv6_address() + @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} + @type ipv6_address :: + {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | ip_address() - @hackney_options [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - @adapter Application.get_env(:tesla, :adapter) + @defaults [pool: :federation] - @doc """ - Configure a client connection + require Logger - # Returns + alias Pleroma.Config + alias Pleroma.HTTP.Adapter - Tesla.Env.client + @doc """ + Merge default connection & adapter options with received ones. """ - @spec new(Keyword.t()) :: Tesla.Env.client() - def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> pool_timeout() + |> Keyword.merge(opts) + |> adapter().options(uri) + end + + defp pool_timeout(opts) do + timeout = + Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) + + Keyword.merge(opts, timeout: timeout) end - # fetch Hackney options - # - def hackney_options(opts) do - options = Keyword.get(opts, :adapter, []) - adapter_options = Pleroma.Config.get([:http, :adapter], []) - proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - - @hackney_options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + @spec after_request(keyword()) :: :ok + def after_request(opts), do: adapter().after_request(opts) + + defp adapter do + case Application.get_env(:tesla, :adapter) do + Tesla.Adapter.Gun -> Adapter.Gun + Tesla.Adapter.Hackney -> Adapter.Hackney + _ -> Adapter + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + :error -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..ad47dc936 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -4,21 +4,47 @@ defmodule Pleroma.HTTP do @moduledoc """ - + Wrapper for `Tesla.request/2`. """ alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder + alias Tesla.Client + alias Tesla.Env + + require Logger @type t :: __MODULE__ @doc """ - Builds and perform http request. + Performs GET request. + + See `Pleroma.HTTP.request/5` + """ + @spec get(Request.url() | nil, Request.headers(), keyword()) :: + nil | {:ok, Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) + def get(nil, _, _), do: nil + def get(url, headers, options), do: request(:get, url, "", headers, options) + + @doc """ + Performs POST request. + + See `Pleroma.HTTP.request/5` + """ + @spec post(Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def post(url, body, headers \\ [], options \\ []), + do: request(:post, url, body, headers, options) + + @doc """ + Builds and performs http request. # Arguments: `method` - :get, :post, :put, :delete - `url` - `body` + `url` - full url + `body` - request body `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` `options` - custom, per-request middleware or adapter options @@ -26,61 +52,97 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - def request(method, url, body \\ "", headers \\ [], options \\ []) do + @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def request(method, url, body, headers, options) when is_binary(url) do + with uri <- URI.parse(url), + received_adapter_opts <- Keyword.get(options, :adapter, []), + adapter_opts <- Connection.options(uri, received_adapter_opts), + options <- put_in(options[:adapter], adapter_opts), + params <- Keyword.get(options, :params, []), + request <- build_request(method, headers, options, url, body, params), + client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), + pid <- Process.whereis(adapter_opts[:pool]) do + pool_alive? = + if tesla_adapter() == Tesla.Adapter.Gun do + if pid, do: Process.alive?(pid), else: false + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = + request( + client, + request, + request_opts + ) + + Connection.after_request(adapter_opts) + + response + end + end + + @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} + def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + + def request(%Client{} = client, request, %{body_as: :chunks}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool_alive?: false}) do + request_try(client, request) + end + + def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do try do - options = - process_request_options(options) - |> process_sni_options(url) - - params = Keyword.get(options, :params, []) - - %{} - |> Builder.method(method) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.url(url) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), + timeout + 1_000 + ) rescue e -> {:error, e} catch + :exit, {:timeout, _} -> + Logger.warn("Receive response from pool failed #{request[:url]}") + {:error, :recv_pool_timeout} + :exit, e -> {:error, e} end end - defp process_sni_options(options, nil), do: options - - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() - - case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options + @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request_try(client, request) do + try do + Tesla.request(client, request) + rescue + e -> + {:error, e} + catch + :exit, e -> + {:error, e} end end - def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + defp build_request(method, headers, options, url, body, params) do + Builder.new() + |> Builder.method(method) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.url(url) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Builder.convert_to_keyword() end - @doc """ - Performs GET request. - - See `Pleroma.HTTP.request/5` - """ - def get(url, headers \\ [], options \\ []), - do: request(:get, url, "", headers, options) - - @doc """ - Performs POST request. - - See `Pleroma.HTTP.request/5` - """ - def post(url, body, headers \\ [], options \\ []), - do: request(:post, url, body, headers, options) + defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex new file mode 100644 index 000000000..891d88d53 --- /dev/null +++ b/lib/pleroma/http/request.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Request do + @moduledoc """ + Request struct. + """ + defstruct method: :get, url: "", query: [], headers: [], body: "", opts: [] + + @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch + @type url :: String.t() + @type headers :: [{String.t(), String.t()}] + + @type t :: %__MODULE__{ + method: method(), + url: url(), + query: keyword(), + headers: headers(), + body: String.t(), + opts: keyword() + } +end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..491acd0f9 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -7,77 +7,54 @@ defmodule Pleroma.HTTP.RequestBuilder do Helper functions for building Tesla requests """ - @doc """ - Specify the request method when building a request - - ## Parameters - - - request (Map) - Collected request options - - m (atom) - Request method + alias Pleroma.HTTP.Request + alias Tesla.Multipart - ## Returns - - Map + @doc """ + Creates new request """ - @spec method(map(), atom) :: map() - def method(request, m) do - Map.put_new(request, :method, m) - end + @spec new(Request.t()) :: Request.t() + def new(%Request{} = request \\ %Request{}), do: request @doc """ Specify the request method when building a request + """ + @spec method(Request.t(), Request.method()) :: Request.t() + def method(request, m), do: %{request | method: m} - ## Parameters - - - request (Map) - Collected request options - - u (String) - Request URL - - ## Returns - - Map + @doc """ + Specify the request method when building a request """ - @spec url(map(), String.t()) :: map() - def url(request, u) do - Map.put_new(request, :url, u) - end + @spec url(Request.t(), Request.url()) :: Request.t() + def url(request, u), do: %{request | url: u} @doc """ Add headers to the request """ - @spec headers(map(), list(tuple)) :: map() - def headers(request, header_list) do - header_list = + @spec headers(Request.t(), Request.headers()) :: Request.t() + def headers(request, headers) do + headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + headers ++ [{"user-agent", Pleroma.Application.user_agent()}] else - header_list + headers end - Map.put_new(request, :headers, header_list) + %{request | headers: headers_list} end @doc """ Add custom, per-request middleware or adapter options to the request """ - @spec opts(map(), Keyword.t()) :: map() - def opts(request, options) do - Map.put_new(request, :opts, options) - end + @spec opts(Request.t(), keyword()) :: Request.t() + def opts(request, options), do: %{request | opts: options} + # NOTE: isn't used anywhere @doc """ Add optional parameters to the request - ## Parameters - - - request (Map) - Collected request options - - definitions (Map) - Map of parameter name to parameter location. - - options (KeywordList) - The provided optional parameters - - ## Returns - - Map """ - @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map() + @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() def add_optional_params(request, _, []), do: request def add_optional_params(request, definitions, [{key, value} | tail]) do @@ -94,49 +71,43 @@ defmodule Pleroma.HTTP.RequestBuilder do @doc """ Add optional parameters to the request - - ## Parameters - - - request (Map) - Collected request options - - location (atom) - Where to put the parameter - - key (atom) - The name of the parameter - - value (any) - The value of the parameter - - ## Returns - - Map """ - @spec add_param(map(), atom, atom, any()) :: map() - def add_param(request, :query, :query, values), do: Map.put(request, :query, values) + @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t() + def add_param(request, :query, :query, values), do: %{request | query: values} - def add_param(request, :body, :body, value), do: Map.put(request, :body, value) + def add_param(request, :body, :body, value), do: %{request | body: value} def add_param(request, :body, key, value) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) + |> Map.put(:body, Multipart.new()) |> Map.update!( :body, - &Tesla.Multipart.add_field( + &Multipart.add_field( &1, key, Jason.encode!(value), - headers: [{:"Content-Type", "application/json"}] + headers: [{"content-type", "application/json"}] ) ) end def add_param(request, :file, name, path) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) - |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name)) + |> Map.put(:body, Multipart.new()) + |> Map.update!(:body, &Multipart.add_file(&1, path, name: name)) end def add_param(request, :form, name, value) do - request - |> Map.update(:body, %{name => value}, &Map.put(&1, name, value)) + Map.update(request, :body, %{name => value}, &Map.put(&1, name, value)) end def add_param(request, location, key, value) do Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}])) end + + def convert_to_keyword(request) do + request + |> Map.from_struct() + |> Enum.into([]) + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 037c42339..5e9bf1574 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -137,7 +137,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -150,7 +150,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -162,7 +162,7 @@ defmodule Pleroma.Object.Fetcher do date = Pleroma.Signature.signed_date() headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex new file mode 100644 index 000000000..0be189304 --- /dev/null +++ b/lib/pleroma/otp_version.ex @@ -0,0 +1,63 @@ +defmodule Pleroma.OTPVersion do + @type check_status() :: :undefined | {:error, String.t()} | :ok + + require Logger + + @spec check_version() :: check_status() + def check_version do + # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version + paths = [ + Path.join(:code.root_dir(), "OTP_VERSION"), + Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) + ] + + :tesla + |> Application.get_env(:adapter) + |> get_and_check_version(paths) + end + + @spec get_and_check_version(module(), [Path.t()]) :: check_status() + def get_and_check_version(Tesla.Adapter.Gun, paths) do + paths + |> check_files() + |> check_version() + end + + def get_and_check_version(_, _), do: :ok + + defp check_files([]), do: nil + + defp check_files([path | paths]) do + if File.exists?(path) do + File.read!(path) + else + check_files(paths) + end + end + + defp check_version(nil), do: :undefined + + defp check_version(version) do + try do + version = String.replace(version, ~r/\r|\n|\s/, "") + + formatted = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + with [major, minor] when length(formatted) == 2 <- formatted, + true <- (major == 22 and minor >= 2) or major > 22 do + :ok + else + false -> {:error, version} + _ -> :undefined + end + rescue + _ -> :undefined + catch + _ -> :undefined + end + end +end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex new file mode 100644 index 000000000..1ed16d1c1 --- /dev/null +++ b/lib/pleroma/pool/connections.ex @@ -0,0 +1,415 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Connections do + use GenServer + + require Logger + + @type domain :: String.t() + @type conn :: Pleroma.Gun.Conn.t() + + @type t :: %__MODULE__{ + conns: %{domain() => conn()}, + opts: keyword() + } + + defstruct conns: %{}, opts: [] + + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + + @spec start_link({atom(), keyword()}) :: {:ok, pid()} + def start_link({name, opts}) do + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + + @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil + def checkin(url, name) + def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) + + def checkin(%URI{} = uri, name) do + timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + + GenServer.call( + name, + {:checkin, uri}, + timeout + ) + end + + @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok + def open_conn(url, name, opts \\ []) + def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) + + def open_conn(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:receive, false) + |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) + end + + @spec alive?(atom()) :: boolean() + def alive?(name) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + + @spec get_state(atom()) :: t() + def get_state(name) do + GenServer.call(name, :state) + end + + @spec checkout(pid(), pid(), atom()) :: :ok + def checkout(conn, pid, name) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + @impl true + def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do + Logger.debug("opening new #{compose_uri(uri)}") + max_connections = state.opts[:max_connections] + + key = compose_key(uri) + + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, state, opts) + else + try_to_open_conn(key, uri, state, opts) + end + end + + @impl true + def handle_cast({:checkout, conn_pid, pid}, state) do + Logger.debug("checkout #{inspect(conn_pid)}") + + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid), + used_by <- List.keydelete(conn.used_by, pid, 0) do + conn_state = + if used_by == [] do + :idle + else + conn.conn_state + end + + put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + else + false -> + Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + state + end + + {:noreply, state} + end + + @impl true + def handle_call({:checkin, uri}, from, state) do + Logger.debug("checkin #{compose_uri(uri)}") + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + Logger.debug("reusing conn #{compose_uri(uri)}") + + with time <- :os.system_time(:second), + last_reference <- time - current_conn.last_reference, + current_crf <- crf(last_reference, 100, current_conn.crf), + state <- + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) do + {:reply, conn, state} + end + + %{gun_state: gun_state} when gun_state == :down -> + {:reply, nil, state} + + nil -> + {:reply, nil, state} + end + end + + @impl true + def handle_call(:state, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do + state = + with true <- Process.alive?(conn_pid), + conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + time <- :os.system_time(:second), + last_reference <- time - conn.last_reference, + current_crf <- crf(last_reference, 100, conn.crf) do + put_in(state.conns[key], %{ + conn + | gun_state: :up, + last_reference: time, + crf: current_crf, + conn_state: :active, + retries: 0 + }) + else + :error_gun_info -> + Logger.warn(":gun.info caused error") + state + + false -> + Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + # we can't get info on this pid, because pid is dead + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid) do + if conn.retries == 5 do + Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + :ok = API.close(conn.conn) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + put_in(state.conns[key], %{ + conn + | gun_state: :down, + retries: conn.retries + 1 + }) + end + else + false -> + # gun can send gun_down for closed conn, maybe connection is not closed yet + Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.warn( + ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" + ) + + :ok = API.close(conn_pid) + + state + end + + {:noreply, state} + end + + defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + + defp compose_key_gun_info(pid) do + try do + # sometimes :gun.info can raise MatchError, which lead to pool terminate + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end + + "#{scheme}:#{host}:#{port}" + rescue + _ -> :error_gun_info + end + end + + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + + defp find_conn(conns, conn_pid, conn_key) do + Enum.find(conns, fn {key, conn} -> + key == conn_key and conn.conn == conn_pid + end) + end + + defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{uri.scheme}://#{ + compose_uri(uri) + }: #{inspect(error)}" + ) + + {:noreply, state} + end + end + + defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do + Logger.debug("opening conn #{compose_uri(uri)}") + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn), + state <- + put_in(state.conns[key], %Conn{ + conn: conn, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + }) do + Logger.debug("new conn opened #{compose_uri(uri)}") + Logger.debug("replying to the call #{compose_uri(uri)}") + {:noreply, state} + else + error -> + Logger.warn( + "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ + inspect(error) + }" + ) + + {:noreply, state} + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + @spec get_unused_conns(map()) :: [{domain(), conn()}] + def get_unused_conns(conns) do + conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + end + + defp try_to_open_conn(key, uri, state, opts) do + Logger.debug("try to open conn #{compose_uri(uri)}") + + with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), + :ok <- API.close(least_used.conn), + state <- + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) do + Logger.debug( + "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" + ) + + open_conn(key, uri, state, opts) + else + [] -> {:noreply, state} + end + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end + + def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" +end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex new file mode 100644 index 000000000..a7ae64ce4 --- /dev/null +++ b/lib/pleroma/pool/pool.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool do + def child_spec(opts) do + poolboy_opts = + opts + |> Keyword.put(:worker_module, Pleroma.Pool.Request) + |> Keyword.put(:name, {:local, opts[:name]}) + |> Keyword.put(:size, opts[:size]) + |> Keyword.put(:max_overflow, opts[:max_overflow]) + + %{ + id: opts[:id] || {__MODULE__, make_ref()}, + start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, + restart: :permanent, + shutdown: 5000, + type: :worker + } + end +end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex new file mode 100644 index 000000000..2c3574561 --- /dev/null +++ b/lib/pleroma/pool/request.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Request do + use GenServer + + require Logger + + def start_link(args) do + GenServer.start_link(__MODULE__, args) + end + + @impl true + def init(_), do: {:ok, []} + + @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: + {:ok, Tesla.Env.t()} | {:error, any()} + def execute(pid, client, request, timeout) do + GenServer.call(pid, {:execute, client, request}, timeout) + end + + @impl true + def handle_call({:execute, client, request}, _from, state) do + response = Pleroma.HTTP.request_try(client, request) + + {:reply, response, state} + end + + @impl true + def handle_info({:gun_data, _conn, stream, _, _}, state) do + # in some cases if we reuse conn and got {:error, :body_too_large} + # gun continues to send messages to this process, + # so we flush messages for this request + :ok = :gun.flush(stream) + + {:noreply, state} + end + + @impl true + def handle_info({:gun_up, _conn, _protocol}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do + # don't flush messages here, because gun can reconnect + {:noreply, state} + end + + @impl true + def handle_info({:gun_error, _conn, stream, _error}, state) do + :ok = :gun.flush(stream) + {:noreply, state} + end + + @impl true + def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info(msg, state) do + Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") + {:noreply, state} + end +end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex new file mode 100644 index 000000000..32be2264d --- /dev/null +++ b/lib/pleroma/pool/supervisor.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Supervisor do + use Supervisor + + alias Pleroma.Pool + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def init(_) do + children = + [ + %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, + [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + } + ] ++ pools() + + Supervisor.init(children, strategy: :one_for_one) + end + + defp pools do + for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pool_opts + |> Keyword.put(:id, {Pool, pool_name}) + |> Keyword.put(:name, pool_name) + |> Pool.child_spec() + end + end +end diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..63261b94c 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,19 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} - @callback stream_body(reference() | pid() | map()) :: - {:ok, binary()} | :done | {:error, String.t()} + @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()} @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + :tesla + |> Application.get_env(:adapter) + |> client() end + + defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney + defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla + defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client) end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e41560ab0 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + @impl true + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + @impl true + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + @impl true + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..55a11b4a8 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Tesla do + @type headers() :: [{String.t(), String.t()}] + @type status() :: pos_integer() + + @behaviour Pleroma.ReverseProxy.Client + + @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: + {:ok, status(), headers} + | {:ok, status(), headers, map()} + | {:error, atom() | String.t()} + | no_return() + + @impl true + def request(method, url, headers, body, opts \\ []) do + _adapter = check_adapter() + + with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), + {:ok, response} <- + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, opts) + ) do + if is_map(response.body) and method != :head do + {:ok, response.status, response.headers, response.body} + else + {:ok, response.status, response.headers} + end + else + {:error, error} -> {:error, error} + end + end + + @impl true + @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + def stream_body(%{pid: pid, opts: opts, fin: true}) do + # if connection was sended and there were redirects, we need to close new conn - pid manually + if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) + # if there were redirects we need to checkout old conn + conn = opts[:old_conn] || opts[:conn] + + if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) + + :done + end + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> + {:ok, body, Map.put(client, :fin, true)} + + {:nofin, part} -> + {:ok, part, client} + + {:error, error} -> + {:error, error} + end + end + + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do + adapter = check_adapter() + adapter.read_chunk(pid, stream, opts) + end + + @impl true + @spec close(map) :: :ok | no_return() + def close(%{pid: pid}) do + adapter = check_adapter() + adapter.close(pid) + end + + defp check_adapter do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter == Tesla.Adapter.Gun do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 2ed719315..9f5710c92 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - alias Pleroma.HTTP - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @@ -61,10 +59,10 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [gun](https://github.com/ninenines/gun). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -97,11 +95,7 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) - |> Keyword.merge(@default_hackney_options) - |> Keyword.merge(Keyword.get(opts, :http, [])) - |> HTTP.process_request_options() + client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, [])) req_headers = build_req_headers(conn.req_headers, opts) @@ -113,7 +107,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -159,11 +153,11 @@ defmodule Pleroma.ReverseProxy do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -213,7 +207,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index df774b0f7..ade87daf2 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ - pool: :media, - recv_timeout: 10_000 + @options [ + pool: :media ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.put(@options, :recv_timeout, 10_000) + else + @options + end + url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: opts) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 16b1a53d2..0ae926375 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] if Pleroma.Config.get(:env) == :test do @@ -25,8 +23,18 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: @hackney_options), + Pleroma.HTTP.get(url, [], adapter: opts), data <- Floki.attribute(html, "link[rel~=me]", "href") ++ Floki.attribute(html, "a[rel~=me]", "href") do diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c06b0a0f2..9deb03845 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] defp parsers do @@ -77,8 +75,18 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) html |> parse_html diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index b4cc80179..91e9e2271 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -205,7 +205,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) -- cgit v1.2.3 From 7d73e7a09a72354acf526652e307149afbf5b1a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 Feb 2020 09:18:09 -0600 Subject: Spelling --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f25afeda7..ec6475e96 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do case Connections.checkin(uri, :gun_connections) do nil -> Logger.info( - "Gun connections pool checkin was not succesfull. Trying to open conn for next request." + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) :ok = Connections.open_conn(uri, :gun_connections, opts) -- cgit v1.2.3 From c9db0507f8d49aee9988b0b63477672f5df9c0b2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 19 Feb 2020 12:19:03 +0300 Subject: removing retry option and changing some logger messages levels --- lib/pleroma/http/adapter/gun.ex | 28 +++++++++++++++++++++------- lib/pleroma/pool/connections.ex | 17 ++++++++--------- 2 files changed, 29 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index ec6475e96..f1018dd8d 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 20_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry_timeout: 100, + retry: 0, await_up_timeout: 5_000 ] @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do try do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.info( + Logger.debug( "Gun connections pool checkin was not successful. Trying to open conn for next request." ) @@ -97,7 +97,9 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri(uri)}") + Logger.debug( + "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" + ) opts |> Keyword.put(:conn, conn) @@ -105,18 +107,30 @@ defmodule Pleroma.HTTP.Adapter.Gun do end rescue error -> - Logger.warn("Gun connections pool checkin caused error #{inspect(error)}") + Logger.warn( + "Gun connections pool checkin caused error #{uri.scheme}://#{ + Connections.compose_uri(uri) + } #{inspect(error)}" + ) + opts catch :exit, {:timeout, _} -> - Logger.info( - "Gun connections pool checkin with timeout error #{Connections.compose_uri(uri)}" + Logger.warn( + "Gun connections pool checkin with timeout error #{uri.scheme}://#{ + Connections.compose_uri(uri) + }" ) opts :exit, error -> - Logger.warn("Gun pool checkin exited with error #{inspect(error)}") + Logger.warn( + "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + inspect(error) + }" + ) + opts end end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 1ed16d1c1..c7136e0e0 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -52,8 +52,7 @@ defmodule Pleroma.Pool.Connections do opts = opts |> Enum.into(%{}) - |> Map.put_new(:receive, false) - |> Map.put_new(:retry, pool_opts[:retry] || 5) + |> Map.put_new(:retry, pool_opts[:retry] || 0) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) @@ -108,11 +107,11 @@ defmodule Pleroma.Pool.Connections do put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else false -> - Logger.warn("checkout for closed conn #{inspect(conn_pid)}") + Logger.debug("checkout for closed conn #{inspect(conn_pid)}") state nil -> - Logger.info("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") state end @@ -172,15 +171,15 @@ defmodule Pleroma.Pool.Connections do }) else :error_gun_info -> - Logger.warn(":gun.info caused error") + Logger.debug(":gun.info caused error") state false -> - Logger.warn(":gun_up message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" ) @@ -216,11 +215,11 @@ defmodule Pleroma.Pool.Connections do else false -> # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.warn(":gun_down message for closed conn #{inspect(conn_pid)}") + Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") state nil -> - Logger.warn( + Logger.debug( ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" ) -- cgit v1.2.3 From a03c420b84d9901be70520d8c027ccb53449990d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:32:42 +0300 Subject: by default don't use gun retries remove conn depends on retry setting from config --- lib/pleroma/pool/connections.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c7136e0e0..d20927580 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Pool.Connections do use GenServer + alias Pleroma.Config + require Logger @type domain :: String.t() @@ -33,7 +35,7 @@ defmodule Pleroma.Pool.Connections do def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Pleroma.Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) GenServer.call( name, @@ -47,7 +49,7 @@ defmodule Pleroma.Pool.Connections do def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) def open_conn(%URI{} = uri, name, opts) do - pool_opts = Pleroma.Config.get([:connections_pool], []) + pool_opts = Config.get([:connections_pool], []) opts = opts @@ -193,12 +195,13 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid) do - if conn.retries == 5 do - Logger.debug("closing conn if retries is eq 5 #{inspect(conn_pid)}") + if conn.retries == retries do + Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) put_in( -- cgit v1.2.3 From ad8f26c0a4a0a579e93547e78313d3e4ecef6ed5 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 12:53:40 +0300 Subject: more info in Connections.checkin timout errors --- lib/pleroma/http/adapter/gun.ex | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index f1018dd8d..fc40b324a 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -115,11 +115,16 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts catch - :exit, {:timeout, _} -> + :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> + messages_len = + :gun_connections + |> Process.whereis() + |> Process.info(:message_queue_len) + Logger.warn( - "Gun connections pool checkin with timeout error #{uri.scheme}://#{ - Connections.compose_uri(uri) - }" + "Gun connections pool checkin with timeout error for #{operation} #{method} #{ + uri.scheme + }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" ) opts -- cgit v1.2.3 From 6806df80ddb1e52aef2b89b923d9a3e2844b5aeb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:28:16 +0300 Subject: don't log info ssl messages --- lib/pleroma/http/adapter/gun.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index fc40b324a..0a6872ad6 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -58,7 +58,8 @@ defmodule Pleroma.HTTP.Adapter.Gun do depth: 20, reuse_sessions: false, verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]} + {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + log_level: :warning ] ] -- cgit v1.2.3 From f604f9e47061b9d47c1bb62cc7aaf44fabdf69b3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 14:33:55 +0300 Subject: hackney pool timeout --- lib/pleroma/http/connection.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 85918341a..e2d7afbbd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -33,8 +33,14 @@ defmodule Pleroma.HTTP.Connection do end defp pool_timeout(opts) do - timeout = - Config.get([:pools, opts[:pool], :timeout]) || Config.get([:pools, :default, :timeout]) + {config_key, default} = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout])} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) Keyword.merge(opts, timeout: timeout) end -- cgit v1.2.3 From d44f9e3b6cfd5a0dae07f6194bfd05360afd6560 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 21 Feb 2020 16:56:55 +0300 Subject: fix for timeout clause --- lib/pleroma/http/adapter/gun.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 0a6872ad6..7b7e38d8c 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -117,7 +117,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts catch :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - messages_len = + {:message_queue_len, messages_len} = :gun_connections |> Process.whereis() |> Process.info(:message_queue_len) -- cgit v1.2.3 From 8efae966b1e87fe448a13d04eae0898c4a102c29 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 24 Feb 2020 19:56:27 +0300 Subject: open conn in separate task --- lib/mix/tasks/pleroma/benchmark.ex | 2 +- lib/pleroma/gun/api.ex | 7 +- lib/pleroma/gun/api/mock.ex | 5 +- lib/pleroma/gun/conn.ex | 146 +++++++++++++++++++ lib/pleroma/gun/gun.ex | 5 +- lib/pleroma/http/adapter/gun.ex | 21 ++- lib/pleroma/pool/connections.ex | 287 +++++++++++++------------------------ 7 files changed, 266 insertions(+), 207 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 01e079136..7a7430289 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -79,7 +79,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do start_pleroma() :ok = - Pleroma.Pool.Connections.open_conn( + Pleroma.Gun.Conn.open( "https://httpbin.org/stream-bytes/1500", :gun_connections ) diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index a0c3c5415..f79c9f443 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -6,9 +6,10 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok - @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} @callback connect(pid(), map()) :: reference() @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok def open(host, port, opts), do: api().open(host, port, opts) @@ -16,11 +17,13 @@ defmodule Pleroma.Gun.API do def close(pid), do: api().close(pid) - def await_up(pid), do: api().await_up(pid) + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) + def set_owner(pid, owner), do: api().set_owner(pid, owner) + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 0134b016e..6d24b0e69 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -118,7 +118,10 @@ defmodule Pleroma.Gun.API.Mock do end @impl API - def await_up(_pid), do: {:ok, :http} + def await_up(_pid, _timeout), do: {:ok, :http} + + @impl API + def set_owner(_pid, _owner), do: :ok @impl API def connect(pid, %{host: _, port: 80}) do diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 2474829d6..ddb9f30b0 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,6 +6,11 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ + alias Pleroma.Gun.API + alias Pleroma.Pool.Connections + + require Logger + @type gun_state :: :up | :down @type conn_state :: :active | :idle @@ -26,4 +31,145 @@ defmodule Pleroma.Gun.Conn do last_reference: 0, crf: 1, retries: 0 + + @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil + def open(url, name, opts \\ []) + def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) + + def open(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:retry, pool_opts[:retry] || 0) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") + + conn_pid = + if Connections.count(name) < opts[:max_connection] do + do_open(uri, opts) + else + try_do_open(name, uri, opts) + end + + if is_pid(conn_pid) do + conn = %Pleroma.Gun.Conn{ + conn: conn_pid, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + } + + :ok = API.set_owner(conn_pid, Process.whereis(name)) + Connections.add_conn(name, key, conn) + end + end + + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with http proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection with socks proxy #{ + Connections.compose_uri_log(uri) + } #{inspect(error)}" + ) + + nil + end + end + + defp do_open(%URI{host: host, port: port} = uri, opts) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + + with {:ok, conn} <- API.open(host, port, opts), + {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" + ) + + nil + end + end + + defp destination_opts(%URI{host: host, port: port}) do + {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + defp try_do_open(name, uri, opts) do + Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") + + with [{close_key, least_used} | _conns] <- + Connections.get_unused_conns(name), + :ok <- Pleroma.Gun.API.close(least_used.conn) do + Connections.remove_conn(name, close_key) + + do_open(uri, opts) + else + [] -> nil + end + end end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 4a1bbc95f..da82983b1 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Gun do defdelegate close(pid), to: :gun @impl API - defdelegate await_up(pid), to: :gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun @impl API defdelegate connect(pid, opts), to: :gun @@ -42,4 +42,7 @@ defmodule Pleroma.Gun do @spec flush(pid() | reference()) :: :ok defdelegate flush(pid), to: :gun + + @impl API + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 7b7e38d8c..908d71898 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -12,7 +12,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do alias Pleroma.Pool.Connections @defaults [ - connect_timeout: 20_000, + connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, retry: 0, @@ -94,13 +94,11 @@ defmodule Pleroma.HTTP.Adapter.Gun do "Gun connections pool checkin was not successful. Trying to open conn for next request." ) - :ok = Connections.open_conn(uri, :gun_connections, opts) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts conn when is_pid(conn) -> - Logger.debug( - "received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}" - ) + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts |> Keyword.put(:conn, conn) @@ -109,13 +107,14 @@ defmodule Pleroma.HTTP.Adapter.Gun do rescue error -> Logger.warn( - "Gun connections pool checkin caused error #{uri.scheme}://#{ - Connections.compose_uri(uri) - } #{inspect(error)}" + "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ + inspect(error) + }" ) opts catch + # TODO: here must be no timeouts :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> {:message_queue_len, messages_len} = :gun_connections @@ -124,15 +123,15 @@ defmodule Pleroma.HTTP.Adapter.Gun do Logger.warn( "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - uri.scheme - }://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" + Connections.compose_uri_log(uri) + }. Messages length: #{messages_len}" ) opts :exit, error -> Logger.warn( - "Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ + "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ inspect(error) }" ) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index d20927580..a444f822f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] alias Pleroma.Gun.API - alias Pleroma.Gun.Conn @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -44,23 +43,6 @@ defmodule Pleroma.Pool.Connections do ) end - @spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok - def open_conn(url, name, opts \\ []) - def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts) - - def open_conn(%URI{} = uri, name, opts) do - pool_opts = Config.get([:connections_pool], []) - - opts = - opts - |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) - |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) - - GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}}) - end - @spec alive?(atom()) :: boolean() def alive?(name) do pid = Process.whereis(name) @@ -72,23 +54,37 @@ defmodule Pleroma.Pool.Connections do GenServer.call(name, :state) end + @spec count(atom()) :: pos_integer() + def count(name) do + GenServer.call(name, :count) + end + + @spec get_unused_conns(atom()) :: [{domain(), conn()}] + def get_unused_conns(name) do + GenServer.call(name, :unused_conns) + end + @spec checkout(pid(), pid(), atom()) :: :ok def checkout(conn, pid, name) do GenServer.cast(name, {:checkout, conn, pid}) end - @impl true - def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do - Logger.debug("opening new #{compose_uri(uri)}") - max_connections = state.opts[:max_connections] + @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok + def add_conn(name, key, conn) do + GenServer.cast(name, {:add_conn, key, conn}) + end - key = compose_key(uri) + @spec remove_conn(atom(), String.t()) :: :ok + def remove_conn(name, key) do + GenServer.cast(name, {:remove_conn, key}) + end - if Enum.count(state.conns) < max_connections do - open_conn(key, uri, state, opts) - else - try_to_open_conn(key, uri, state, opts) - end + @impl true + def handle_cast({:add_conn, key, conn}, state) do + state = put_in(state.conns[key], conn) + + Process.monitor(conn.conn) + {:noreply, state} end @impl true @@ -120,14 +116,20 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end + @impl true + def handle_cast({:remove_conn, key}, state) do + state = put_in(state.conns, Map.delete(state.conns, key)) + {:noreply, state} + end + @impl true def handle_call({:checkin, uri}, from, state) do - Logger.debug("checkin #{compose_uri(uri)}") - key = compose_key(uri) + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + Logger.debug("checkin #{key}") case state.conns[key] do %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> - Logger.debug("reusing conn #{compose_uri(uri)}") + Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), last_reference <- time - current_conn.last_reference, @@ -154,12 +156,31 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_call(:state, _from, state), do: {:reply, state, state} + @impl true + def handle_call(:count, _from, state) do + {:reply, Enum.count(state.conns), state} + end + + @impl true + def handle_call(:unused_conns, _from, state) do + unused_conns = + state.conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf <= y.crf and x.last_reference <= y.last_reference + end) + + {:reply, unused_conns, state} + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with true <- Process.alive?(conn_pid), - conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), + with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + {true, key} <- {Process.alive?(conn_pid), key}, time <- :os.system_time(:second), last_reference <- time - conn.last_reference, current_crf <- crf(last_reference, 100, conn.crf) do @@ -176,15 +197,17 @@ defmodule Pleroma.Pool.Connections do Logger.debug(":gun.info caused error") state - false -> + {false, key} -> Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - state - nil -> - Logger.debug( - ":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state" + put_in( + state.conns, + Map.delete(state.conns, key) ) + nil -> + Logger.debug(":gun_up message for conn which is not found in state") + :ok = API.close(conn_pid) state @@ -198,8 +221,8 @@ defmodule Pleroma.Pool.Connections do retries = Config.get([:connections_pool, :retry], 0) # we can't get info on this pid, because pid is dead state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid) do + with {key, conn} <- find_conn(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = API.close(conn.conn) @@ -216,16 +239,18 @@ defmodule Pleroma.Pool.Connections do }) end else - false -> + {false, key} -> # gun can send gun_down for closed conn, maybe connection is not closed yet Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - state - nil -> - Logger.debug( - ":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state" + put_in( + state.conns, + Map.delete(state.conns, key) ) + nil -> + Logger.debug(":gun_down message for conn which is not found in state") + :ok = API.close(conn_pid) state @@ -234,7 +259,29 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end - defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" + @impl true + def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do + Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") + + state = + with {key, conn} <- find_conn(state.conns, conn_pid) do + Enum.each(conn.used_by, fn {pid, _ref} -> + Process.exit(pid, reason) + end) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + nil -> + Logger.debug(":DOWN message for conn which is not found in state") + + state + end + + {:noreply, state} + end defp compose_key_gun_info(pid) do try do @@ -265,153 +312,11 @@ defmodule Pleroma.Pool.Connections do end) end - defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do - connect_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - - with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with http proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do - version = - proxy_type - |> to_string() - |> String.last() - |> case do - "4" -> 4 - _ -> 5 - end - - socks_opts = - uri - |> destination_opts() - |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - |> Map.put(:version, version) - - opts = - opts - |> Map.put(:protocols, [:socks]) - |> Map.put(:socks_opts, socks_opts) - - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection with socks proxy #{uri.scheme}://#{ - compose_uri(uri) - }: #{inspect(error)}" - ) - - {:noreply, state} - end - end - - defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do - Logger.debug("opening conn #{compose_uri(uri)}") - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn), - state <- - put_in(state.conns[key], %Conn{ - conn: conn, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - }) do - Logger.debug("new conn opened #{compose_uri(uri)}") - Logger.debug("replying to the call #{compose_uri(uri)}") - {:noreply, state} - else - error -> - Logger.warn( - "Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{ - inspect(error) - }" - ) - - {:noreply, state} - end - end - - defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) - %{host: host, port: port} - end - - defp add_http2_opts(opts, "https", tls_opts) do - Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) - end - - defp add_http2_opts(opts, _, _), do: opts - - @spec get_unused_conns(map()) :: [{domain(), conn()}] - def get_unused_conns(conns) do - conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) - end - - defp try_to_open_conn(key, uri, state, opts) do - Logger.debug("try to open conn #{compose_uri(uri)}") - - with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns), - :ok <- API.close(least_used.conn), - state <- - put_in( - state.conns, - Map.delete(state.conns, close_key) - ) do - Logger.debug( - "least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}" - ) - - open_conn(key, uri, state, opts) - else - [] -> {:noreply, state} - end - end - def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end -- cgit v1.2.3 From 4c8569d403f47957f7a5d698c595959007c8a95a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 12:19:29 +0300 Subject: otp_version refactor --- lib/pleroma/application.ex | 35 ++++++++++++---------- lib/pleroma/otp_version.ex | 74 ++++++++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 57 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00e33d7ac..9b228d6b9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -66,16 +66,23 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] - case Pleroma.OTPVersion.check_version() do - :ok -> :ok - {:error, version} -> raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - :undefined -> raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " + if adapter() == Tesla.Adapter.Gun do + case Pleroma.OTPVersion.check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -202,11 +209,7 @@ defmodule Pleroma.Application do [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_) do - :tesla - |> Application.get_env(:adapter) - |> http_pools() - end + defp http_pools_children(_), do: http_pools(adapter()) defp http_pools(Tesla.Adapter.Hackney) do pools = [:federation, :media] @@ -227,4 +230,6 @@ defmodule Pleroma.Application do defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] defp http_pools(_), do: [] + + defp adapter, do: Application.get_env(:tesla, :adapter) end diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 0be189304..54ceaff47 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -1,63 +1,53 @@ -defmodule Pleroma.OTPVersion do - @type check_status() :: :undefined | {:error, String.t()} | :ok +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only - require Logger +defmodule Pleroma.OTPVersion do + @type check_status() :: :ok | :undefined | {:error, String.t()} - @spec check_version() :: check_status() - def check_version do + @spec check() :: check_status() + def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version - paths = [ + [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] - - :tesla - |> Application.get_env(:adapter) - |> get_and_check_version(paths) + |> get_version_from_files() + |> do_check() end - @spec get_and_check_version(module(), [Path.t()]) :: check_status() - def get_and_check_version(Tesla.Adapter.Gun, paths) do + @spec check([Path.t()]) :: check_status() + def check(paths) do paths - |> check_files() - |> check_version() + |> get_version_from_files() + |> do_check() end - def get_and_check_version(_, _), do: :ok - - defp check_files([]), do: nil + defp get_version_from_files([]), do: nil - defp check_files([path | paths]) do + defp get_version_from_files([path | paths]) do if File.exists?(path) do File.read!(path) else - check_files(paths) + get_version_from_files(paths) end end - defp check_version(nil), do: :undefined - - defp check_version(version) do - try do - version = String.replace(version, ~r/\r|\n|\s/, "") - - formatted = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - with [major, minor] when length(formatted) == 2 <- formatted, - true <- (major == 22 and minor >= 2) or major > 22 do - :ok - else - false -> {:error, version} - _ -> :undefined - end - rescue - _ -> :undefined - catch - _ -> :undefined + defp do_check(nil), do: :undefined + + defp do_check(version) do + version = String.replace(version, ~r/\r|\n|\s/, "") + + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor >= 2) or major > 22 do + :ok + else + {:error, version} end end end -- cgit v1.2.3 From 097ad10d02598fb6b77f305c10341a13fb57ceee Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:51 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index a444f822f..c5098cd86 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -128,7 +128,7 @@ defmodule Pleroma.Pool.Connections do Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") with time <- :os.system_time(:second), -- cgit v1.2.3 From 2c8d80dc0ad594cfe25ebadd9e7a187c95914b34 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:29:57 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c5098cd86..c4c5fd66c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -145,7 +145,7 @@ defmodule Pleroma.Pool.Connections do {:reply, conn, state} end - %{gun_state: gun_state} when gun_state == :down -> + %{gun_state: :down} -> {:reply, nil, state} nil -> -- cgit v1.2.3 From a3ad028973154dafad910d4d73d7d4d4822627c1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:34:36 +0000 Subject: Apply suggestion to lib/pleroma/http/adapter.ex --- lib/pleroma/http/adapter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 6166a3eb4..32046b1d3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -57,7 +57,7 @@ defmodule Pleroma.HTTP.Adapter do {:error, :einval} -> {:domain, :idna.encode(charlist)} - {:ok, ip} when is_tuple(ip) and tuple_size(ip) in [4, 8] -> + {:ok, ip} -> {:ip, ip} end end -- cgit v1.2.3 From d30ff35d94ff7d8bc07f0221323a75b07641ee8d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:46:53 +0000 Subject: Apply suggestion to lib/pleroma/http/request_builder.ex --- lib/pleroma/http/request_builder.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 491acd0f9..046741d99 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -35,7 +35,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, headers) do headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - headers ++ [{"user-agent", Pleroma.Application.user_agent()}] + [{"user-agent", Pleroma.Application.user_agent()} | headers] else headers end -- cgit v1.2.3 From 614e3934f9190ff199df087de34146ad5f34c660 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:50:42 +0000 Subject: Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ad47dc936..5fb468689 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -64,8 +64,8 @@ defmodule Pleroma.HTTP do client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), pid <- Process.whereis(adapter_opts[:pool]) do pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun do - if pid, do: Process.alive?(pid), else: false + if tesla_adapter() == Tesla.Adapter.Gun && pid do + Process.alive?(pid) else false end -- cgit v1.2.3 From a21a66972f8733de766bc538fe81f2e0ccb57925 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:52:01 +0000 Subject: Apply suggestion to lib/pleroma/http/http.ex --- lib/pleroma/http/http.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 5fb468689..0235f89ea 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -76,12 +76,7 @@ defmodule Pleroma.HTTP do |> Map.put(:env, Pleroma.Config.get([:env])) |> Map.put(:pool_alive?, pool_alive?) - response = - request( - client, - request, - request_opts - ) + response = request(client, request, request_opts) Connection.after_request(adapter_opts) -- cgit v1.2.3 From 7eb65929924af50146d89192c2cf557e3bdbf07f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:31 +0000 Subject: Apply suggestion to lib/pleroma/pool/connections.ex --- lib/pleroma/pool/connections.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index c4c5fd66c..84617815f 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -180,10 +180,10 @@ defmodule Pleroma.Pool.Connections do state = with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid, conn_key), - {true, key} <- {Process.alive?(conn_pid), key}, - time <- :os.system_time(:second), - last_reference <- time - conn.last_reference, - current_crf <- crf(last_reference, 100, conn.crf) do + {true, key} <- {Process.alive?(conn_pid), key} do + time = :os.system_time(:second) + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From 151dc4e387cfbb91b7cd85461ce0deb1e5f5fe30 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 09:53:37 +0000 Subject: Apply suggestion to lib/pleroma/reverse_proxy/client/tesla.ex --- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 55a11b4a8..498a905e1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -16,7 +16,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true def request(method, url, headers, body, opts \\ []) do - _adapter = check_adapter() + check_adapter() with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), {:ok, response} <- -- cgit v1.2.3 From 28ed4b41d03c6a137d198b8c67fb081c7ebfbbc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:05:28 +0300 Subject: naming for checkin from pool timeout --- lib/pleroma/pool/connections.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 84617815f..05fa8f7ad 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Pool.Connections do def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :receive_connection_timeout], 250) + timeout = Config.get([:connections_pool, :checkin_timeout], 250) GenServer.call( name, @@ -184,6 +184,7 @@ defmodule Pleroma.Pool.Connections do time = :os.system_time(:second) last_reference = time - conn.last_reference current_crf = crf(last_reference, 100, conn.crf) + put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From 24d1ac125c6ae719b3d119f2ec0079dcd74eadc2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 13:24:19 +0300 Subject: hiding raise error logic to otp_version module --- lib/pleroma/application.ex | 23 ++++------------------- lib/pleroma/otp_version.ex | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9b228d6b9..d0b9c3c41 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,10 @@ defmodule Pleroma.Application do setup_instrumenters() load_custom_modules() + if adapter() == Tesla.Adapter.Gun do + Pleroma.OTPVersion.check!() + end + # Define workers and child supervisors to be supervised children = [ @@ -66,25 +70,6 @@ defmodule Pleroma.Application do Pleroma.Gopher.Server ] - if adapter() == Tesla.Adapter.Gun do - case Pleroma.OTPVersion.check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 54ceaff47..9ced2d27d 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -5,6 +5,26 @@ defmodule Pleroma.OTPVersion do @type check_status() :: :ok | :undefined | {:error, String.t()} + @spec check!() :: :ok | no_return() + def check! do + case check() do + :ok -> + :ok + + {:error, version} -> + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + + :undefined -> + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + end + @spec check() :: check_status() def check do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version -- cgit v1.2.3 From d0e4d3ca3b9d8b8ed00d58e9e1c2a05ab561326c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 14:56:49 +0300 Subject: removing unnecessary with comment in tesla client impovement --- lib/pleroma/pool/connections.ex | 40 +++++++++++++++---------------- lib/pleroma/reverse_proxy/client/tesla.ex | 8 ++++--- 2 files changed, 25 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 05fa8f7ad..bde3ffd13 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -36,17 +36,16 @@ defmodule Pleroma.Pool.Connections do def checkin(%URI{} = uri, name) do timeout = Config.get([:connections_pool, :checkin_timeout], 250) - GenServer.call( - name, - {:checkin, uri}, - timeout - ) + GenServer.call(name, {:checkin, uri}, timeout) end @spec alive?(atom()) :: boolean() def alive?(name) do - pid = Process.whereis(name) - if pid, do: Process.alive?(pid), else: false + if pid = Process.whereis(name) do + Process.alive?(pid) + else + false + end end @spec get_state(atom()) :: t() @@ -131,19 +130,20 @@ defmodule Pleroma.Pool.Connections do %{conn: conn, gun_state: :up} = current_conn -> Logger.debug("reusing conn #{key}") - with time <- :os.system_time(:second), - last_reference <- time - current_conn.last_reference, - current_crf <- crf(last_reference, 100, current_conn.crf), - state <- - put_in(state.conns[key], %{ - current_conn - | last_reference: time, - crf: current_crf, - conn_state: :active, - used_by: [from | current_conn.used_by] - }) do - {:reply, conn, state} - end + time = :os.system_time(:second) + last_reference = time - current_conn.last_reference + current_crf = crf(last_reference, 100, current_conn.crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) + + {:reply, conn, state} %{gun_state: :down} -> {:reply, nil, state} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 498a905e1..80a0c8972 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -18,8 +18,9 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - with opts <- Keyword.merge(opts, body_as: :chunks, mode: :passive), - {:ok, response} <- + opts = Keyword.merge(opts, body_as: :chunks) + + with {:ok, response} <- Pleroma.HTTP.request( method, url, @@ -40,7 +41,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done def stream_body(%{pid: pid, opts: opts, fin: true}) do - # if connection was sended and there were redirects, we need to close new conn - pid manually + # if connection was reused, but in tesla were redirects, + # tesla returns new opened connection, which must be closed manually if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) # if there were redirects we need to checkout old conn conn = opts[:old_conn] || opts[:conn] -- cgit v1.2.3 From 05429730e46b8605544637feebd4c409a4e9ed18 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:11:48 +0300 Subject: unnecessary with --- lib/pleroma/http/http.ex | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 0235f89ea..f7b0095d7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -55,33 +55,36 @@ defmodule Pleroma.HTTP do @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do - with uri <- URI.parse(url), - received_adapter_opts <- Keyword.get(options, :adapter, []), - adapter_opts <- Connection.options(uri, received_adapter_opts), - options <- put_in(options[:adapter], adapter_opts), - params <- Keyword.get(options, :params, []), - request <- build_request(method, headers, options, url, body, params), - client <- Tesla.client([Tesla.Middleware.FollowRedirects], tesla_adapter()), - pid <- Process.whereis(adapter_opts[:pool]) do - pool_alive? = - if tesla_adapter() == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end - - request_opts = - adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) - - response = request(client, request, request_opts) - - Connection.after_request(adapter_opts) - - response - end + uri = URI.parse(url) + received_adapter_opts = Keyword.get(options, :adapter, []) + adapter_opts = Connection.options(uri, received_adapter_opts) + options = put_in(options[:adapter], adapter_opts) + params = Keyword.get(options, :params, []) + request = build_request(method, headers, options, url, body, params) + + adapter = Application.get_env(:tesla, :adapter) + client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) + + pid = Process.whereis(adapter_opts[:pool]) + + pool_alive? = + if adapter == Tesla.Adapter.Gun && pid do + Process.alive?(pid) + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = request(client, request, request_opts) + + Connection.after_request(adapter_opts) + + response end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} @@ -138,6 +141,4 @@ defmodule Pleroma.HTTP do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end - - defp tesla_adapter, do: Application.get_env(:tesla, :adapter) end -- cgit v1.2.3 From ee8071f0d5a8a53f6a9ae635d6ea57ce8576e21b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:12:09 +0300 Subject: removing unused method --- lib/pleroma/http/request_builder.ex | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 046741d99..5b92ce764 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -49,26 +49,6 @@ defmodule Pleroma.HTTP.RequestBuilder do @spec opts(Request.t(), keyword()) :: Request.t() def opts(request, options), do: %{request | opts: options} - # NOTE: isn't used anywhere - @doc """ - Add optional parameters to the request - - """ - @spec add_optional_params(Request.t(), %{optional(atom) => atom}, keyword()) :: map() - def add_optional_params(request, _, []), do: request - - def add_optional_params(request, definitions, [{key, value} | tail]) do - case definitions do - %{^key => location} -> - request - |> add_param(location, key, value) - |> add_optional_params(definitions, tail) - - _ -> - add_optional_params(request, definitions, tail) - end - end - @doc """ Add optional parameters to the request """ -- cgit v1.2.3 From e605e79df9761cef3d9f93c489dd4618c6b70eda Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 15:44:13 +0300 Subject: simplification of formatting host method case for format_proxy method --- lib/pleroma/gun/conn.ex | 6 +++--- lib/pleroma/http/adapter.ex | 29 +++-------------------------- lib/pleroma/http/adapter/gun.ex | 20 ++++++++++++++++---- 3 files changed, 22 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index ddb9f30b0..a33d75558 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do @@ -131,7 +131,7 @@ defmodule Pleroma.Gun.Conn do end defp do_open(%URI{host: host, port: port} = uri, opts) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) with {:ok, conn} <- API.open(host, port, opts), {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do @@ -149,7 +149,7 @@ defmodule Pleroma.Gun.Conn do end defp destination_opts(%URI{host: host, port: port}) do - {_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host) + host = Pleroma.HTTP.Connection.parse_host(host) %{host: host, port: port} end diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex index 32046b1d3..a3b84d8f3 100644 --- a/lib/pleroma/http/adapter.ex +++ b/lib/pleroma/http/adapter.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter do @@ -8,7 +8,6 @@ defmodule Pleroma.HTTP.Adapter do @type proxy :: {Connection.host(), pos_integer()} | {Connection.proxy_type(), pos_integer()} - @type host_type :: :domain | :ip @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok @@ -29,9 +28,8 @@ defmodule Pleroma.HTTP.Adapter do def format_proxy(nil), do: nil def format_proxy(proxy_url) do - with {:ok, host, port} <- Connection.parse_proxy(proxy_url) do - {host, port} - else + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil end @@ -40,25 +38,4 @@ defmodule Pleroma.HTTP.Adapter do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) - - @spec domain_or_fallback(String.t()) :: charlist() - def domain_or_fallback(host) do - case domain_or_ip(host) do - {:domain, domain} -> domain - {:ip, _ip} -> to_charlist(host) - end - end - - @spec domain_or_ip(String.t()) :: {host_type(), Connection.host()} - def domain_or_ip(host) do - charlist = to_charlist(host) - - case :inet.parse_address(charlist) do - {:error, :einval} -> - {:domain, :idna.encode(charlist)} - - {:ok, ip} -> - {:ip, ip} - end - end end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 908d71898..5e88786bd 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Adapter.Gun do @@ -42,7 +42,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do end defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = Adapter.domain_or_fallback(host) + formatted_host = format_host(host) Keyword.put(opts, :original, "#{formatted_host}:#{port}") end @@ -57,8 +57,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, [check_hostname: Adapter.domain_or_fallback(host)]}, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, log_level: :warning ] ] @@ -139,4 +138,17 @@ defmodule Pleroma.HTTP.Adapter.Gun do opts end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end -- cgit v1.2.3 From 7d68924e4f7233590457aa7e32a21f082dd0584f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:08:21 +0300 Subject: naming --- lib/pleroma/gun/conn.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a33d75558..a8b8c92c1 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Gun.Conn do if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) else - try_do_open(name, uri, opts) + close_least_used_and_do_open(name, uri, opts) end if is_pid(conn_pid) do @@ -159,7 +159,7 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts - defp try_do_open(name, uri, opts) do + defp close_least_used_and_do_open(name, uri, opts) do Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") with [{close_key, least_used} | _conns] <- -- cgit v1.2.3 From 8fc00b7cbff86885ec99d01821c403a766202659 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:27:46 +0300 Subject: return error if connection failed to open --- lib/pleroma/gun/conn.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a8b8c92c1..9ae419092 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Gun.Conn do } #{inspect(error)}" ) - nil + error end end @@ -126,7 +126,7 @@ defmodule Pleroma.Gun.Conn do } #{inspect(error)}" ) - nil + error end end @@ -144,7 +144,7 @@ defmodule Pleroma.Gun.Conn do }" ) - nil + error end end @@ -169,7 +169,7 @@ defmodule Pleroma.Gun.Conn do do_open(uri, opts) else - [] -> nil + [] -> {:error, :pool_overflowed} end end end -- cgit v1.2.3 From 6ebf389d6e6ca5f3e56f9b017531f5f7e301ed3c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 16:51:49 +0300 Subject: poolboy timeout fix --- lib/pleroma/http/http.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index f7b0095d7..4b774472e 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -102,8 +102,8 @@ defmodule Pleroma.HTTP do try do :poolboy.transaction( pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout + 500), - timeout + 1_000 + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout ) rescue e -> -- cgit v1.2.3 From aaa879ce75a62e69a458226e65bef31b0f2ed08c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:22 +0300 Subject: proxy parsing errors --- lib/pleroma/http/connection.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index e2d7afbbd..bdd062929 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ defmodule Pleroma.HTTP.Connection do else {_, _} -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} :error -> Logger.warn("parsing port in proxy fail #{inspect(proxy)}") - {:error, :error_parsing_port_in_proxy} + {:error, :invalid_proxy_port} _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Connection do else _ -> Logger.warn("parsing proxy fail #{inspect(proxy)}") - {:error, :error_parsing_proxy} + {:error, :invalid_proxy} end end -- cgit v1.2.3 From 24bf5c4e89e6f97ed3d53157cead48c04015a51b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:27:56 +0300 Subject: remove try block from pool request --- lib/pleroma/http/http.ex | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 4b774472e..cc0c39400 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -99,23 +99,11 @@ defmodule Pleroma.HTTP do end def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - try do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout - ) - rescue - e -> - {:error, e} - catch - :exit, {:timeout, _} -> - Logger.warn("Receive response from pool failed #{request[:url]}") - {:error, :recv_pool_timeout} - - :exit, e -> - {:error, e} - end + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout + ) end @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} -- cgit v1.2.3 From 1ad34bfdbaee7d98167dc7dc7be8b65fd5e6c5f1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 17:44:04 +0300 Subject: no try block in checkout connection --- lib/pleroma/http/adapter/gun.ex | 53 +++++++---------------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 5e88786bd..30c5c3c16 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -86,56 +86,21 @@ defmodule Pleroma.HTTP.Adapter.Gun do end defp try_to_get_conn(uri, opts) do - try do - case Connections.checkin(uri, :gun_connections) do - nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) - opts - - conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - end - rescue - error -> - Logger.warn( - "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.debug( + "Gun connections pool checkin was not successful. Trying to open conn for next request." ) + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) opts - catch - # TODO: here must be no timeouts - :exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> - {:message_queue_len, messages_len} = - :gun_connections - |> Process.whereis() - |> Process.info(:message_queue_len) - - Logger.warn( - "Gun connections pool checkin with timeout error for #{operation} #{method} #{ - Connections.compose_uri_log(uri) - }. Messages length: #{messages_len}" - ) - opts - - :exit, error -> - Logger.warn( - "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" - ) + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) end end -- cgit v1.2.3 From 8854770fc4e9079131a0897d5fb6c0ccccf98bc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:01:35 +0300 Subject: retry and retry_timeout settings default change --- lib/pleroma/gun/conn.ex | 4 ++-- lib/pleroma/http/adapter/gun.ex | 3 ++- lib/pleroma/pool/connections.ex | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 9ae419092..d73bec360 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -42,8 +42,8 @@ defmodule Pleroma.Gun.Conn do opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 0) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100) + |> Map.put_new(:retry, pool_opts[:retry] || 1) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex index 30c5c3c16..ecf9c5b62 100644 --- a/lib/pleroma/http/adapter/gun.ex +++ b/lib/pleroma/http/adapter/gun.ex @@ -15,7 +15,8 @@ defmodule Pleroma.HTTP.Adapter.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 0, + retry: 1, + retry_timeout: 1000, await_up_timeout: 5_000 ] diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index bde3ffd13..0f7a1bfd8 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -219,7 +219,7 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 0) + retries = Config.get([:connections_pool, :retry], 1) # we can't get info on this pid, because pid is dead state = with {key, conn} <- find_conn(state.conns, conn_pid), -- cgit v1.2.3 From f98ee730f01de528797e38f27964b69a465662c4 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 18:53:44 +0300 Subject: adapter renaming to adapter_helper --- lib/pleroma/http/adapter.ex | 41 ---------- lib/pleroma/http/adapter/gun.ex | 120 ----------------------------- lib/pleroma/http/adapter/hackney.ex | 41 ---------- lib/pleroma/http/adapter_helper.ex | 41 ++++++++++ lib/pleroma/http/adapter_helper/gun.ex | 120 +++++++++++++++++++++++++++++ lib/pleroma/http/adapter_helper/hackney.ex | 41 ++++++++++ lib/pleroma/http/connection.ex | 8 +- 7 files changed, 206 insertions(+), 206 deletions(-) delete mode 100644 lib/pleroma/http/adapter.ex delete mode 100644 lib/pleroma/http/adapter/gun.ex delete mode 100644 lib/pleroma/http/adapter/hackney.ex create mode 100644 lib/pleroma/http/adapter_helper.ex create mode 100644 lib/pleroma/http/adapter_helper/gun.ex create mode 100644 lib/pleroma/http/adapter_helper/hackney.ex (limited to 'lib') diff --git a/lib/pleroma/http/adapter.ex b/lib/pleroma/http/adapter.ex deleted file mode 100644 index a3b84d8f3..000000000 --- a/lib/pleroma/http/adapter.ex +++ /dev/null @@ -1,41 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Adapter do - alias Pleroma.HTTP.Connection - - @type proxy :: - {Connection.host(), pos_integer()} - | {Connection.proxy_type(), pos_integer()} - - @callback options(keyword(), URI.t()) :: keyword() - @callback after_request(keyword()) :: :ok - - @spec options(keyword(), URI.t()) :: keyword() - def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - maybe_add_proxy(opts, format_proxy(proxy)) - end - - @spec maybe_get_conn(URI.t(), keyword()) :: keyword() - def maybe_get_conn(_uri, opts), do: opts - - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok - - @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil - def format_proxy(nil), do: nil - - def format_proxy(proxy_url) do - case Connection.parse_proxy(proxy_url) do - {:ok, host, port} -> {host, port} - {:ok, type, host, port} -> {type, host, port} - _ -> nil - end - end - - @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() - def maybe_add_proxy(opts, nil), do: opts - def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) -end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex deleted file mode 100644 index ecf9c5b62..000000000 --- a/lib/pleroma/http/adapter/gun.ex +++ /dev/null @@ -1,120 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Adapter.Gun do - @behaviour Pleroma.HTTP.Adapter - - alias Pleroma.HTTP.Adapter - - require Logger - - alias Pleroma.Pool.Connections - - @defaults [ - connect_timeout: 5_000, - domain_lookup_timeout: 5_000, - tls_handshake_timeout: 5_000, - retry: 1, - retry_timeout: 1000, - await_up_timeout: 5_000 - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - - @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> add_original(uri) - |> add_scheme_opts(uri) - |> Adapter.maybe_add_proxy(Adapter.format_proxy(proxy)) - |> maybe_get_conn(uri, connection_opts) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts) do - with conn when not is_nil(conn) <- opts[:conn], - body_as when body_as != :chunks <- opts[:body_as] do - Connections.checkout(conn, self(), :gun_connections) - end - - :ok - end - - defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = format_host(host) - - Keyword.put(opts, :original, "#{formatted_host}:#{port}") - end - - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do - adapter_opts = [ - certificates_verification: true, - tls_opts: [ - verify: :verify_peer, - cacertfile: CAStore.file_path(), - depth: 20, - reuse_sessions: false, - verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, - log_level: :warning - ] - ] - - adapter_opts = - if port != 443 do - Keyword.put(adapter_opts, :transport, :tls) - else - adapter_opts - end - - Keyword.merge(opts, adapter_opts) - end - - defp maybe_get_conn(adapter_opts, uri, connection_opts) do - {receive_conn?, opts} = - adapter_opts - |> Keyword.merge(connection_opts) - |> Keyword.pop(:receive_conn, true) - - if Connections.alive?(:gun_connections) and receive_conn? do - try_to_get_conn(uri, opts) - else - opts - end - end - - defp try_to_get_conn(uri, opts) do - case Connections.checkin(uri, :gun_connections) do - nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) - opts - - conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - end - end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end -end diff --git a/lib/pleroma/http/adapter/hackney.ex b/lib/pleroma/http/adapter/hackney.ex deleted file mode 100644 index 00db30083..000000000 --- a/lib/pleroma/http/adapter/hackney.ex +++ /dev/null @@ -1,41 +0,0 @@ -defmodule Pleroma.HTTP.Adapter.Hackney do - @behaviour Pleroma.HTTP.Adapter - - @defaults [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - - @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> Keyword.merge(connection_opts) - |> add_scheme_opts(uri) - |> Pleroma.HTTP.Adapter.maybe_add_proxy(proxy) - end - - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do - ssl_opts = [ - ssl_options: [ - # Workaround for remote server certificate chain issues - partial_chain: &:hackney_connect.partial_chain/1, - - # We don't support TLS v1.3 yet - versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], - server_name_indication: to_charlist(host) - ] - ] - - Keyword.merge(opts, ssl_opts) - end - - def after_request(_), do: :ok -end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex new file mode 100644 index 000000000..2c13666ec --- /dev/null +++ b/lib/pleroma/http/adapter_helper.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), pos_integer()} + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex new file mode 100644 index 000000000..b3298ec7f --- /dev/null +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -0,0 +1,120 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper.Gun do + @behaviour Pleroma.HTTP.AdapterHelper + + alias Pleroma.HTTP.AdapterHelper + + require Logger + + alias Pleroma.Pool.Connections + + @defaults [ + connect_timeout: 5_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry: 1, + retry_timeout: 1000, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> add_original(uri) + |> add_scheme_opts(uri) + |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) + |> maybe_get_conn(uri, connection_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + with conn when not is_nil(conn) <- opts[:conn], + body_as when body_as != :chunks <- opts[:body_as] do + Connections.checkout(conn, self(), :gun_connections) + end + + :ok + end + + defp add_original(opts, %URI{host: host, port: port}) do + formatted_host = format_host(host) + + Keyword.put(opts, :original, "#{formatted_host}:#{port}") + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + adapter_opts = [ + certificates_verification: true, + tls_opts: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, + log_level: :warning + ] + ] + + adapter_opts = + if port != 443 do + Keyword.put(adapter_opts, :transport, :tls) + else + adapter_opts + end + + Keyword.merge(opts, adapter_opts) + end + + defp maybe_get_conn(adapter_opts, uri, connection_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(connection_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + try_to_get_conn(uri, opts) + else + opts + end + end + + defp try_to_get_conn(uri, opts) do + case Connections.checkin(uri, :gun_connections) do + nil -> + Logger.debug( + "Gun connections pool checkin was not successful. Trying to open conn for next request." + ) + + Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) + opts + + conn when is_pid(conn) -> + Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") + + opts + |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end +end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex new file mode 100644 index 000000000..a0e161eaa --- /dev/null +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.HTTP.AdapterHelper.Hackney do + @behaviour Pleroma.HTTP.AdapterHelper + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + + @defaults + |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index bdd062929..dc2761182 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.Connection do require Logger alias Pleroma.Config - alias Pleroma.HTTP.Adapter + alias Pleroma.HTTP.AdapterHelper @doc """ Merge default connection & adapter options with received ones. @@ -50,9 +50,9 @@ defmodule Pleroma.HTTP.Connection do defp adapter do case Application.get_env(:tesla, :adapter) do - Tesla.Adapter.Gun -> Adapter.Gun - Tesla.Adapter.Hackney -> Adapter.Hackney - _ -> Adapter + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper end end -- cgit v1.2.3 From 884d9710b209cc9981c7de61d4e95fd26cd83820 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Mar 2020 19:24:14 +0300 Subject: refactoring for gun api modules --- lib/pleroma/gun/api.ex | 46 ++++++++---- lib/pleroma/gun/api/mock.ex | 154 ---------------------------------------- lib/pleroma/gun/conn.ex | 22 +++--- lib/pleroma/gun/gun.ex | 49 ++++--------- lib/pleroma/pool/connections.ex | 10 +-- 5 files changed, 62 insertions(+), 219 deletions(-) delete mode 100644 lib/pleroma/gun/api/mock.ex (limited to 'lib') diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f79c9f443..76aac5874 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -3,27 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do - @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} - @callback info(pid()) :: map() - @callback close(pid()) :: :ok - @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} - @callback connect(pid(), map()) :: reference() - @callback await(pid(), reference()) :: {:response, :fin, 200, []} - @callback set_owner(pid(), pid()) :: :ok + @behaviour Pleroma.Gun - def open(host, port, opts), do: api().open(host, port, opts) + alias Pleroma.Gun - def info(pid), do: api().info(pid) + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] - def close(pid), do: api().close(pid) + @impl Gun + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) - def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) + @impl Gun + defdelegate info(pid), to: :gun - def connect(pid, opts), do: api().connect(pid, opts) + @impl Gun + defdelegate close(pid), to: :gun - def await(pid, ref), do: api().await(pid, ref) + @impl Gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun - def set_owner(pid, owner), do: api().set_owner(pid, owner) + @impl Gun + defdelegate connect(pid, opts), to: :gun - defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) + @impl Gun + defdelegate await(pid, ref), to: :gun + + @impl Gun + defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex deleted file mode 100644 index 6d24b0e69..000000000 --- a/lib/pleroma/gun/api/mock.ex +++ /dev/null @@ -1,154 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Gun.API.Mock do - @behaviour Pleroma.Gun.API - - alias Pleroma.Gun.API - - @impl API - def open('some-domain.com', 443, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'some-domain.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open(ip, port, _) - when ip in [{10_755, 10_368, 61_708, 131, 64_206, 45_068, 0, 9_694}, {127, 0, 0, 1}] and - port in [80, 443] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - scheme = if port == 443, do: "https", else: "http" - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: scheme, - origin_host: ip, - origin_port: port - }) - - {:ok, conn_pid} - end - - @impl API - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks5, 'localhost', 1234}, - socks_opts: %{host: 'proxy-socks.com', port: 80, version: 5} - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "http", - origin_host: 'proxy-socks.com', - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl API - def open('localhost', 1234, %{ - protocols: [:socks], - proxy: {:socks4, 'localhost', 1234}, - socks_opts: %{ - host: 'proxy-socks.com', - port: 443, - protocols: [:http2], - tls_opts: [], - transport: :tls, - version: 4 - } - }) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'proxy-socks.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open('gun-not-up.com', 80, _opts), do: {:error, :timeout} - - @impl API - def open('example.com', port, _) when port in [443, 115] do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "https", - origin_host: 'example.com', - origin_port: 443 - }) - - {:ok, conn_pid} - end - - @impl API - def open(domain, 80, _) do - {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - - Registry.register(API.Mock, conn_pid, %{ - origin_scheme: "http", - origin_host: domain, - origin_port: 80 - }) - - {:ok, conn_pid} - end - - @impl API - def open({127, 0, 0, 1}, 8123, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl API - def open('localhost', 9050, _) do - Task.start_link(fn -> Process.sleep(1_000) end) - end - - @impl API - def await_up(_pid, _timeout), do: {:ok, :http} - - @impl API - def set_owner(_pid, _owner), do: :ok - - @impl API - def connect(pid, %{host: _, port: 80}) do - ref = make_ref() - Registry.register(API.Mock, ref, pid) - ref - end - - @impl API - def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do - ref = make_ref() - Registry.register(API.Mock, ref, pid) - ref - end - - @impl API - def await(pid, ref) do - [{_, ^pid}] = Registry.lookup(API.Mock, ref) - {:response, :fin, 200, []} - end - - @impl API - def info(pid) do - [{_, info}] = Registry.lookup(API.Mock, pid) - info - end - - @impl API - def close(_pid), do: :ok -end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index d73bec360..319718690 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ - alias Pleroma.Gun.API + alias Pleroma.Gun alias Pleroma.Pool.Connections require Logger @@ -65,7 +65,7 @@ defmodule Pleroma.Gun.Conn do last_reference: :os.system_time(:second) } - :ok = API.set_owner(conn_pid, Process.whereis(name)) + :ok = Gun.set_owner(conn_pid, Process.whereis(name)) Connections.add_conn(name, key, conn) end end @@ -77,10 +77,10 @@ defmodule Pleroma.Gun.Conn do |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) with open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream) do + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + stream <- Gun.connect(conn, connect_opts), + {:response, :fin, 200, _} <- Gun.await(conn, stream) do conn else error -> @@ -115,8 +115,8 @@ defmodule Pleroma.Gun.Conn do |> Map.put(:protocols, [:socks]) |> Map.put(:socks_opts, socks_opts) - with {:ok, conn} <- API.open(proxy_host, proxy_port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -133,8 +133,8 @@ defmodule Pleroma.Gun.Conn do defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.Connection.parse_host(host) - with {:ok, conn} <- API.open(host, port, opts), - {:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do + with {:ok, conn} <- Gun.open(host, port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do conn else error -> @@ -164,7 +164,7 @@ defmodule Pleroma.Gun.Conn do with [{close_key, least_used} | _conns] <- Connections.get_unused_conns(name), - :ok <- Pleroma.Gun.API.close(least_used.conn) do + :ok <- Gun.close(least_used.conn) do Connections.remove_conn(name, close_key) do_open(uri, opts) diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index da82983b1..35390bb11 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -3,46 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do - @behaviour Pleroma.Gun.API + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok - alias Pleroma.Gun.API + def open(host, port, opts), do: api().open(host, port, opts) - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :socks_opts, - :ws_opts - ] + def info(pid), do: api().info(pid) - @impl API - def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + def close(pid), do: api().close(pid) - @impl API - defdelegate info(pid), to: :gun + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) - @impl API - defdelegate close(pid), to: :gun + def connect(pid, opts), do: api().connect(pid, opts) - @impl API - defdelegate await_up(pid, timeout \\ 5_000), to: :gun + def await(pid, ref), do: api().await(pid, ref) - @impl API - defdelegate connect(pid, opts), to: :gun + def set_owner(pid, owner), do: api().set_owner(pid, owner) - @impl API - defdelegate await(pid, ref), to: :gun - - @spec flush(pid() | reference()) :: :ok - defdelegate flush(pid), to: :gun - - @impl API - defdelegate set_owner(pid, owner), to: :gun + defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 0f7a1bfd8..92179fbfc 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun.API + alias Pleroma.Gun @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do @@ -209,7 +209,7 @@ defmodule Pleroma.Pool.Connections do nil -> Logger.debug(":gun_up message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -226,7 +226,7 @@ defmodule Pleroma.Pool.Connections do {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") - :ok = API.close(conn.conn) + :ok = Gun.close(conn.conn) put_in( state.conns, @@ -252,7 +252,7 @@ defmodule Pleroma.Pool.Connections do nil -> Logger.debug(":gun_down message for conn which is not found in state") - :ok = API.close(conn_pid) + :ok = Gun.close(conn_pid) state end @@ -287,7 +287,7 @@ defmodule Pleroma.Pool.Connections do defp compose_key_gun_info(pid) do try do # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = API.info(pid) + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) host = case :inet.ntoa(origin_host) do -- cgit v1.2.3 From d9c5ae7c09c7cbf3f4f66e01b7ed69a3d6388916 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 3 Mar 2020 17:16:24 -0600 Subject: Update Copyrights for gun related files --- lib/pleroma/gun/api.ex | 2 +- lib/pleroma/gun/gun.ex | 2 +- lib/pleroma/http/request.ex | 2 +- lib/pleroma/pool/connections.ex | 2 +- lib/pleroma/pool/pool.ex | 2 +- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/pool/supervisor.ex | 2 +- lib/pleroma/reverse_proxy/client/hackney.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index 76aac5874..f51cd7db8 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.API do diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 35390bb11..81855e89e 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun do diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex index 891d88d53..761bd6ccf 100644 --- a/lib/pleroma/http/request.ex +++ b/lib/pleroma/http/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.Request do diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 92179fbfc..f1fab2a24 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Connections do diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex index a7ae64ce4..21a6fbbc5 100644 --- a/lib/pleroma/pool/pool.ex +++ b/lib/pleroma/pool/pool.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool do diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 2c3574561..cce309599 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Request do diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 32be2264d..f436849ac 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Pool.Supervisor do diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e41560ab0..e84118a90 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Hackney do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 80a0c8972..dbc6b66a3 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do -- cgit v1.2.3 From 6b2fb9160cd945cdd4b1265c793d1f85d559fccb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:23:42 +0300 Subject: otp version --- lib/pleroma/application.ex | 20 ++++++++++++++- lib/pleroma/otp_version.ex | 61 ++++++---------------------------------------- 2 files changed, 27 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d0b9c3c41..c8a0617a5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,7 +43,25 @@ defmodule Pleroma.Application do load_custom_modules() if adapter() == Tesla.Adapter.Gun do - Pleroma.OTPVersion.check!() + if version = Pleroma.OTPVersion.version() do + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor < 2) or major < 22 do + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + end + else + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end end # Define workers and child supervisors to be supervised diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex index 9ced2d27d..114d0054f 100644 --- a/lib/pleroma/otp_version.ex +++ b/lib/pleroma/otp_version.ex @@ -3,71 +3,26 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.OTPVersion do - @type check_status() :: :ok | :undefined | {:error, String.t()} - - @spec check!() :: :ok | no_return() - def check! do - case check() do - :ok -> - :ok - - {:error, version} -> - raise " - !!!OTP VERSION WARNING!!! - You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. - " - - :undefined -> - raise " - !!!OTP VERSION WARNING!!! - To support correct handling of unordered certificates chains - OTP version must be > 22.2. - " - end - end - - @spec check() :: check_status() - def check do + @spec version() :: String.t() | nil + def version do # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version [ Path.join(:code.root_dir(), "OTP_VERSION"), Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) ] |> get_version_from_files() - |> do_check() - end - - @spec check([Path.t()]) :: check_status() - def check(paths) do - paths - |> get_version_from_files() - |> do_check() end - defp get_version_from_files([]), do: nil + @spec get_version_from_files([Path.t()]) :: String.t() | nil + def get_version_from_files([]), do: nil - defp get_version_from_files([path | paths]) do + def get_version_from_files([path | paths]) do if File.exists?(path) do - File.read!(path) + path + |> File.read!() + |> String.replace(~r/\r|\n|\s/, "") else get_version_from_files(paths) end end - - defp do_check(nil), do: :undefined - - defp do_check(version) do - version = String.replace(version, ~r/\r|\n|\s/, "") - - [major, minor] = - version - |> String.split(".") - |> Enum.map(&String.to_integer/1) - |> Enum.take(2) - - if (major == 22 and minor >= 2) or major > 22 do - :ok - else - {:error, version} - end - end end -- cgit v1.2.3 From 22d52f5691d985e7daaa955e97e0722f038f6fae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 09:41:23 +0300 Subject: same copyright date format --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index b3547ecd4..0270b96ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f67f48ab6..fc3475048 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do -- cgit v1.2.3 From d6bebd4f9c8086dd87c75f3637a5d392a05f2daf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 4 Mar 2020 18:13:24 +0300 Subject: moving some logic to tesla adapter - checking original inside gun adapter - flushing streams on max_body error --- lib/pleroma/http/adapter_helper/gun.ex | 17 ++--------------- lib/pleroma/pool/request.ex | 10 ++-------- 2 files changed, 4 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index b3298ec7f..5d5870d90 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -26,7 +26,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @defaults |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) - |> add_original(uri) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) |> maybe_get_conn(uri, connection_opts) @@ -42,17 +41,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do :ok end - defp add_original(opts, %URI{host: host, port: port}) do - formatted_host = format_host(host) - - Keyword.put(opts, :original, "#{formatted_host}:#{port}") - end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host, port: port}) do + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do adapter_opts = [ certificates_verification: true, + transport: :tls, tls_opts: [ verify: :verify_peer, cacertfile: CAStore.file_path(), @@ -63,13 +57,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do ] ] - adapter_opts = - if port != 443 do - Keyword.put(adapter_opts, :transport, :tls) - else - adapter_opts - end - Keyword.merge(opts, adapter_opts) end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index cce309599..0f271b3d0 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -28,12 +28,7 @@ defmodule Pleroma.Pool.Request do end @impl true - def handle_info({:gun_data, _conn, stream, _, _}, state) do - # in some cases if we reuse conn and got {:error, :body_too_large} - # gun continues to send messages to this process, - # so we flush messages for this request - :ok = :gun.flush(stream) - + def handle_info({:gun_data, _conn, _stream, _, _}, state) do {:noreply, state} end @@ -49,8 +44,7 @@ defmodule Pleroma.Pool.Request do end @impl true - def handle_info({:gun_error, _conn, stream, _error}, state) do - :ok = :gun.flush(stream) + def handle_info({:gun_error, _conn, _stream, _error}, state) do {:noreply, state} end -- cgit v1.2.3 From eb324467d9c5c761a776ffc98347246c61ad02ae Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 09:51:52 +0300 Subject: removing try block in getting gun info --- lib/pleroma/pool/connections.ex | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f1fab2a24..f96c08f21 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -285,20 +285,15 @@ defmodule Pleroma.Pool.Connections do end defp compose_key_gun_info(pid) do - try do - # sometimes :gun.info can raise MatchError, which lead to pool terminate - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end + %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - "#{scheme}:#{host}:#{port}" - rescue - _ -> :error_gun_info - end + host = + case :inet.ntoa(origin_host) do + {:error, :einval} -> origin_host + ip -> ip + end + + "#{scheme}:#{host}:#{port}" end defp find_conn(conns, conn_pid) do -- cgit v1.2.3 From f0753eed0fdddd30e127213c89a118dd2e087dc9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 17:31:06 +0300 Subject: removing try block in tesla request added mocks for tests which fail with Tesla.Mock.Error --- lib/pleroma/http/http.ex | 24 +++++------------------- lib/pleroma/pool/request.ex | 2 +- lib/pleroma/web/push/impl.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 3 ++- 4 files changed, 9 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 7b7c79b64..466a94adc 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -88,15 +88,11 @@ defmodule Pleroma.HTTP do end @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request_try(client, request) + def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - def request(%Client{} = client, request, %{body_as: :chunks}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - def request(%Client{} = client, request, %{pool_alive?: false}) do - request_try(client, request) - end + def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do :poolboy.transaction( @@ -106,18 +102,8 @@ defmodule Pleroma.HTTP do ) end - @spec request_try(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} - def request_try(client, request) do - try do - Tesla.request(client, request) - rescue - e -> - {:error, e} - catch - :exit, e -> - {:error, e} - end - end + @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request(client, request), do: Tesla.request(client, request) defp build_request(method, headers, options, url, body, params) do Builder.new() diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index 0f271b3d0..db7c10c01 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Pool.Request do @impl true def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request_try(client, request) + response = Pleroma.HTTP.request(client, request) {:reply, response, state} end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index afa510f08..233e55f21 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Web.Push.Impl do type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) + object = Object.normalize(activity) || activity user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index db567a02e..7ffd0e51b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -173,7 +173,8 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body, status: status}} when status in 200..299 <- + HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} -- cgit v1.2.3 From 058c9b01ac063f3cca22a653032663916a16a234 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 5 Mar 2020 18:28:04 +0300 Subject: returning, not needed --- lib/pleroma/web/push/impl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 233e55f21..afa510f08 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Web.Push.Impl do type = Activity.mastodon_notification_type(notif.activity) gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) - object = Object.normalize(activity) || activity + object = Object.normalize(activity) user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) -- cgit v1.2.3 From c93c3096d5ffb2df1493f2b8e3f0627d9a8c5910 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:04:18 +0300 Subject: little refactor --- lib/pleroma/gun/gun.ex | 6 ++++-- lib/pleroma/http/adapter_helper/gun.ex | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex index 81855e89e..4043e4880 100644 --- a/lib/pleroma/gun/gun.ex +++ b/lib/pleroma/gun/gun.ex @@ -11,6 +11,10 @@ defmodule Pleroma.Gun do @callback await(pid(), reference()) :: {:response, :fin, 200, []} @callback set_owner(pid(), pid()) :: :ok + @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) + + defp api, do: @api + def open(host, port, opts), do: api().open(host, port, opts) def info(pid), do: api().info(pid) @@ -24,6 +28,4 @@ defmodule Pleroma.Gun do def await(pid, ref), do: api().await(pid, ref) def set_owner(pid, owner), do: api().set_owner(pid, owner) - - defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 5d5870d90..9b03f4653 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,10 +5,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - alias Pleroma.HTTP.AdapterHelper - require Logger + alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections @defaults [ @@ -22,20 +21,23 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + formatted_proxy = + Pleroma.Config.get([:http, :proxy_url], nil) + |> AdapterHelper.format_proxy() + + config_opts = Pleroma.Config.get([:http, :adapter], []) @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(AdapterHelper.format_proxy(proxy)) + |> AdapterHelper.maybe_add_proxy(formatted_proxy) |> maybe_get_conn(uri, connection_opts) end @spec after_request(keyword()) :: :ok def after_request(opts) do - with conn when not is_nil(conn) <- opts[:conn], - body_as when body_as != :chunks <- opts[:body_as] do - Connections.checkout(conn, self(), :gun_connections) + if opts[:conn] && opts[:body_as] != :chunks do + Connections.checkout(opts[:conn], self(), :gun_connections) end :ok -- cgit v1.2.3 From 78282dc9839dbd17c4649cd3936bb8f4c8283745 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 6 Mar 2020 21:24:19 +0300 Subject: little polishing --- lib/pleroma/http/adapter_helper/gun.ex | 4 ++-- lib/pleroma/http/adapter_helper/hackney.ex | 4 +++- lib/pleroma/http/connection.ex | 15 ++++++++------- lib/pleroma/pool/connections.ex | 3 +-- 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 9b03f4653..862e851c0 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,11 +5,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper - require Logger - alias Pleroma.HTTP.AdapterHelper alias Pleroma.Pool.Connections + require Logger + @defaults [ connect_timeout: 5_000, domain_lookup_timeout: 5_000, diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index a0e161eaa..d08afae0c 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -13,8 +13,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do def options(connection_opts \\ [], %URI{} = uri) do proxy = Pleroma.Config.get([:http, :proxy_url], nil) + config_opts = Pleroma.Config.get([:http, :adapter], []) + @defaults - |> Keyword.merge(Pleroma.Config.get([:http, :adapter], [])) + |> Keyword.merge(config_opts) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 97eec88c1..777e5d4c8 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -6,6 +6,14 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + + require Logger + + @defaults [pool: :federation] + @type ip_address :: ipv4_address() | ipv6_address() @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} @type ipv6_address :: @@ -13,13 +21,6 @@ defmodule Pleroma.HTTP.Connection do @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | ip_address() - @defaults [pool: :federation] - - require Logger - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - @doc """ Merge default connection & adapter options with received ones. """ diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index f96c08f21..7529e9240 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Pool.Connections do use GenServer alias Pleroma.Config + alias Pleroma.Gun require Logger @@ -19,8 +20,6 @@ defmodule Pleroma.Pool.Connections do defstruct conns: %{}, opts: [] - alias Pleroma.Gun - @spec start_link({atom(), keyword()}) :: {:ok, pid()} def start_link({name, opts}) do GenServer.start_link(__MODULE__, opts, name: name) -- cgit v1.2.3 From 5f42ecc4c74172b1b17c126106fda9da24065b11 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 7 Mar 2020 12:24:39 +0300 Subject: start gun upload pool, if proxy_remote is enabled --- lib/pleroma/pool/supervisor.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index f436849ac..8dc5b64b7 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Pool.Supervisor do use Supervisor + alias Pleroma.Config alias Pleroma.Pool def start_link(args) do @@ -17,8 +18,7 @@ defmodule Pleroma.Pool.Supervisor do %{ id: Pool.Connections, start: - {Pool.Connections, :start_link, - [{:gun_connections, Pleroma.Config.get([:connections_pool])}]} + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} } ] ++ pools() @@ -26,7 +26,16 @@ defmodule Pleroma.Pool.Supervisor do end defp pools do - for {pool_name, pool_opts} <- Pleroma.Config.get([:pools]) do + pools = Config.get(:pools) + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) == false do + Keyword.delete(pools, :upload) + else + pools + end + + for {pool_name, pool_opts} <- pools do pool_opts |> Keyword.put(:id, {Pool, pool_name}) |> Keyword.put(:name, pool_name) -- cgit v1.2.3 From 426f5ee48a09dbf321c013db08cc849c8929d86d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:31:44 +0300 Subject: tesla adapter can't be changed in adminFE --- lib/pleroma/config/transfer_task.ex | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index bf1b943d8..4a4c022f0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -20,8 +20,7 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :markup}, {:pleroma, :streamer}, {:pleroma, :pools}, - {:pleroma, :connections_pool}, - {:tesla, :adapter} + {:pleroma, :connections_pool} ] @reboot_time_subkeys [ @@ -35,8 +34,6 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :gopher, [:enabled]} ] - @reject [nil, :prometheus] - def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) @@ -45,35 +42,30 @@ defmodule Pleroma.Config.TransferTask do @spec load_and_update_env([ConfigDB.t()]) :: :ok | false def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {:configurable, true} <- - {:configurable, Pleroma.Config.get(:configurable_from_database)}, - true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), - started_applications <- Application.started_applications() do + with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) with_deleted = in_db ++ deleted - reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] - - applications = - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in reject_for_restart)) + # TODO: some problem with prometheus after restart! + reject = [nil, :prometheus] - # to be ensured that pleroma will be restarted last - applications = - if :pleroma in applications do - List.delete(applications, :pleroma) ++ [:pleroma] + reject_for_restart = + if restart_pleroma? do + reject else - Restarter.Pleroma.rebooted() - applications + [:pleroma | reject] end - Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) + started_applications = Application.started_applications() + + with_deleted + |> Enum.map(&merge_and_update(&1)) + |> Enum.uniq() + |> Enum.reject(&(&1 in reject_for_restart)) + |> maybe_set_pleroma_last() + |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) :ok else @@ -81,6 +73,18 @@ defmodule Pleroma.Config.TransferTask do end end + defp maybe_set_pleroma_last(apps) do + # to be ensured that pleroma will be restarted last + if :pleroma in apps do + apps + |> List.delete(:pleroma) + |> List.insert_at(-1, :pleroma) + else + Restarter.Pleroma.rebooted() + apps + end + end + defp group_for_restart(:logger, key, _, merged_value) do # change logger configuration in runtime, without restart if Keyword.keyword?(merged_value) and @@ -93,14 +97,10 @@ defmodule Pleroma.Config.TransferTask do nil end - defp group_for_restart(:tesla, _, _, _), do: :pleroma - defp group_for_restart(group, _, _, _) when group != :pleroma, do: group defp group_for_restart(group, key, value, _) do - if pleroma_need_restart?(group, key, value) do - group - end + if pleroma_need_restart?(group, key, value), do: group end defp merge_and_update(setting) do -- cgit v1.2.3 From f39e1b9eff859c0795911212c59304f68fca92bc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 10 Mar 2020 15:54:11 +0300 Subject: add verify tls_opts only when we open connection for other requests tesla will add tls_opts --- lib/pleroma/gun/conn.ex | 24 ++++++++++++++++++++++++ lib/pleroma/http/adapter_helper/gun.ex | 33 +++++---------------------------- lib/pleroma/http/connection.ex | 13 +++++++++++++ 3 files changed, 42 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 319718690..57a847c30 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -45,6 +45,7 @@ defmodule Pleroma.Gun.Conn do |> Map.put_new(:retry, pool_opts[:retry] || 1) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> maybe_add_tls_opts(uri) key = "#{uri.scheme}:#{uri.host}:#{uri.port}" @@ -70,6 +71,29 @@ defmodule Pleroma.Gun.Conn do end end + defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts + + defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + tls_opts = [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, + [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + ] + + tls_opts = + if Keyword.keyword?(opts[:tls_opts]) do + Keyword.merge(tls_opts, opts[:tls_opts]) + else + tls_opts + end + + Map.put(opts, :tls_opts, tls_opts) + end + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do connect_opts = uri diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 862e851c0..55c2b192a 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -45,21 +45,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do - adapter_opts = [ - certificates_verification: true, - transport: :tls, - tls_opts: [ - verify: :verify_peer, - cacertfile: CAStore.file_path(), - depth: 20, - reuse_sessions: false, - verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: format_host(host)]}, - log_level: :warning - ] - ] - - Keyword.merge(opts, adapter_opts) + defp add_scheme_opts(opts, %URI{scheme: "https"}) do + opts + |> Keyword.put(:certificates_verification, true) + |> Keyword.put(:transport, :tls) + |> Keyword.put(:tls_opts, log_level: :warning) end defp maybe_get_conn(adapter_opts, uri, connection_opts) do @@ -93,17 +83,4 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.put(:close_conn, false) end end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 777e5d4c8..0fc88f708 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -106,4 +106,17 @@ defmodule Pleroma.HTTP.Connection do {:ok, ip} -> ip end end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end -- cgit v1.2.3 From 863ec33ba2a90708d199f18683ffe0c4658c710a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 11 Mar 2020 12:21:44 +0100 Subject: Add support for funkwhale Audio activity reel2bits fixture not included as it lacks the Actor fixture for it. Closes: https://git.pleroma.social/pleroma/pleroma/issues/1624 Closes: https://git.pleroma.social/pleroma/pleroma/issues/764 --- lib/pleroma/web/activity_pub/transmogrifier.ex | 5 +++-- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..f52b065f6 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -229,7 +229,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do + def fix_url(%{"type" => object_type, "url" => url} = object) + when object_type in ["Video", "Audio"] and is_list(url) do first_element = Enum.at(url, 0) link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) @@ -398,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f7469cdff..a042075f5 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -421,7 +421,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render_content(%{data: %{"type" => object_type}} = object) - when object_type in ["Video", "Event"] do + when object_type in ["Video", "Event", "Audio"] do with name when not is_nil(name) and name != "" <- object.data["name"] do "

#{name}

#{object.data["content"]}" else -- cgit v1.2.3 From 1306b92997dc6e76e5d617d529dbc229d5aee200 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 12 Mar 2020 18:28:54 +0300 Subject: clean up --- lib/pleroma/application.ex | 18 +++---- lib/pleroma/config/transfer_task.ex | 42 ++++++--------- lib/pleroma/gun/conn.ex | 31 +++++------ lib/pleroma/http/adapter_helper.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 33 +++++------- lib/pleroma/http/connection.ex | 8 +-- lib/pleroma/http/http.ex | 5 +- lib/pleroma/pool/connections.ex | 94 +++++++++++----------------------- 8 files changed, 88 insertions(+), 145 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c8a0617a5..55b5be488 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,7 +42,9 @@ defmodule Pleroma.Application do setup_instrumenters() load_custom_modules() - if adapter() == Tesla.Adapter.Gun do + adapter = Application.get_env(:tesla, :adapter) + + if adapter == Tesla.Adapter.Gun do if version = Pleroma.OTPVersion.version() do [major, minor] = version @@ -74,7 +76,7 @@ defmodule Pleroma.Application do Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - http_pools_children(Config.get(:env)) ++ + http_children(adapter, @env) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -206,15 +208,13 @@ defmodule Pleroma.Application do end # start hackney and gun pools in tests - defp http_pools_children(:test) do + defp http_children(_, :test) do hackney_options = Config.get([:hackney_pools, :federation]) hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) [hackney_pool, Pleroma.Pool.Supervisor] end - defp http_pools_children(_), do: http_pools(adapter()) - - defp http_pools(Tesla.Adapter.Hackney) do + defp http_children(Tesla.Adapter.Hackney, _) do pools = [:federation, :media] pools = @@ -230,9 +230,7 @@ defmodule Pleroma.Application do end end - defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] - - defp http_pools(_), do: [] + defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] - defp adapter, do: Application.get_env(:tesla, :adapter) + defp http_children(_, _), do: [] end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 4a4c022f0..b6d80adb7 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Config.TransferTask do use Task + alias Pleroma.Config alias Pleroma.ConfigDB alias Pleroma.Repo @@ -36,36 +37,31 @@ defmodule Pleroma.Config.TransferTask do def start_link(_) do load_and_update_env() - if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) + if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end - @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do + @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok + def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do + with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) - - with_deleted = in_db ++ deleted # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus] - - reject_for_restart = + reject_restart = if restart_pleroma? do - reject + [nil, :prometheus] else - [:pleroma | reject] + [:pleroma, nil, :prometheus] end started_applications = Application.started_applications() - with_deleted - |> Enum.map(&merge_and_update(&1)) + (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.map(&merge_and_update/1) |> Enum.uniq() - |> Enum.reject(&(&1 in reject_for_restart)) + |> Enum.reject(&(&1 in reject_restart)) |> maybe_set_pleroma_last() - |> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) + |> Enum.each(&restart(started_applications, &1, Config.get(:env))) :ok else @@ -108,18 +104,14 @@ defmodule Pleroma.Config.TransferTask do key = ConfigDB.from_string(setting.key) group = ConfigDB.from_string(setting.group) - default = Pleroma.Config.Holder.config(group, key) + default = Config.Holder.config(group, key) value = ConfigDB.from_binary(setting.value) merged_value = - if Ecto.get_meta(setting, :state) == :deleted do - default - else - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + cond do + Ecto.get_meta(setting, :state) == :deleted -> default + can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) + true -> value end :ok = update_env(group, key, merged_value) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 57a847c30..20823a765 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -49,8 +49,6 @@ defmodule Pleroma.Gun.Conn do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}") - conn_pid = if Connections.count(name) < opts[:max_connection] do do_open(uri, opts) @@ -109,9 +107,9 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection with http proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -145,9 +143,9 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection with socks proxy #{ - Connections.compose_uri_log(uri) - } #{inspect(error)}" + "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" ) error @@ -163,9 +161,7 @@ defmodule Pleroma.Gun.Conn do else error -> Logger.warn( - "Received error on opening connection #{Connections.compose_uri_log(uri)} #{ - inspect(error) - }" + "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) error @@ -184,16 +180,17 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts defp close_least_used_and_do_open(name, uri, opts) do - Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") - - with [{close_key, least_used} | _conns] <- - Connections.get_unused_conns(name), - :ok <- Gun.close(least_used.conn) do - Connections.remove_conn(name, close_key) + with [{key, conn} | _conns] <- Connections.get_unused_conns(name), + :ok <- Gun.close(conn.conn) do + Connections.remove_conn(name, key) do_open(uri, opts) else [] -> {:error, :pool_overflowed} end end + + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 2c13666ec..510722ff9 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @type proxy :: {Connection.host(), pos_integer()} - | {Connection.proxy_type(), pos_integer()} + | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 55c2b192a..f14b95c19 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -20,8 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do ] @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - formatted_proxy = + def options(incoming_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) |> AdapterHelper.format_proxy() @@ -30,8 +30,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @defaults |> Keyword.merge(config_opts) |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(formatted_proxy) - |> maybe_get_conn(uri, connection_opts) + |> AdapterHelper.maybe_add_proxy(proxy) + |> maybe_get_conn(uri, incoming_opts) end @spec after_request(keyword()) :: :ok @@ -43,44 +43,35 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do :ok end - defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + defp add_scheme_opts(opts, %{scheme: "http"}), do: opts - defp add_scheme_opts(opts, %URI{scheme: "https"}) do + defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:transport, :tls) |> Keyword.put(:tls_opts, log_level: :warning) end - defp maybe_get_conn(adapter_opts, uri, connection_opts) do + defp maybe_get_conn(adapter_opts, uri, incoming_opts) do {receive_conn?, opts} = adapter_opts - |> Keyword.merge(connection_opts) + |> Keyword.merge(incoming_opts) |> Keyword.pop(:receive_conn, true) if Connections.alive?(:gun_connections) and receive_conn? do - try_to_get_conn(uri, opts) + checkin_conn(uri, opts) else opts end end - defp try_to_get_conn(uri, opts) do + defp checkin_conn(uri, opts) do case Connections.checkin(uri, :gun_connections) do nil -> - Logger.debug( - "Gun connections pool checkin was not successful. Trying to open conn for next request." - ) - - Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end) + Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) opts conn when is_pid(conn) -> - Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") - - opts - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) + Keyword.merge(opts, conn: conn, close_conn: false) end end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 0fc88f708..76de3fcfe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -71,15 +71,15 @@ defmodule Pleroma.HTTP.Connection do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + Logger.warn("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Connection do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("parsing proxy fail #{inspect(proxy)}") + Logger.warn("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 466a94adc..583b56484 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -56,10 +56,9 @@ defmodule Pleroma.HTTP do {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - received_adapter_opts = Keyword.get(options, :adapter, []) - adapter_opts = Connection.options(uri, received_adapter_opts) + adapter_opts = Connection.options(uri, options[:adapter] || []) options = put_in(options[:adapter], adapter_opts) - params = Keyword.get(options, :params, []) + params = options[:params] || [] request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 7529e9240..772833509 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -87,18 +87,11 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_cast({:checkout, conn_pid, pid}, state) do - Logger.debug("checkout #{inspect(conn_pid)}") - state = with true <- Process.alive?(conn_pid), {key, conn} <- find_conn(state.conns, conn_pid), used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = - if used_by == [] do - :idle - else - conn.conn_state - end + conn_state = if used_by == [], do: :idle, else: conn.conn_state put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) else @@ -123,26 +116,23 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_call({:checkin, uri}, from, state) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - Logger.debug("checkin #{key}") case state.conns[key] do - %{conn: conn, gun_state: :up} = current_conn -> - Logger.debug("reusing conn #{key}") - + %{conn: pid, gun_state: :up} = conn -> time = :os.system_time(:second) - last_reference = time - current_conn.last_reference - current_crf = crf(last_reference, 100, current_conn.crf) + last_reference = time - conn.last_reference + crf = crf(last_reference, 100, conn.crf) state = put_in(state.conns[key], %{ - current_conn + conn | last_reference: time, - crf: current_crf, + crf: crf, conn_state: :active, - used_by: [from | current_conn.used_by] + used_by: [from | conn.used_by] }) - {:reply, conn, state} + {:reply, pid, state} %{gun_state: :down} -> {:reply, nil, state} @@ -164,50 +154,48 @@ defmodule Pleroma.Pool.Connections do def handle_call(:unused_conns, _from, state) do unused_conns = state.conns - |> Enum.filter(fn {_k, v} -> - v.conn_state == :idle and v.used_by == [] - end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf <= y.crf and x.last_reference <= y.last_reference - end) + |> Enum.filter(&filter_conns/1) + |> Enum.sort(&sort_conns/2) {:reply, unused_conns, state} end + defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true + defp filter_conns(_), do: false + + defp sort_conns({_, c1}, {_, c2}) do + c1.crf <= c2.crf and c1.last_reference <= c2.last_reference + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + state = - with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid, conn_key), + with {_key, conn} <- find_conn(state.conns, conn_pid, key), {true, key} <- {Process.alive?(conn_pid), key} do - time = :os.system_time(:second) - last_reference = time - conn.last_reference - current_crf = crf(last_reference, 100, conn.crf) - put_in(state.conns[key], %{ conn | gun_state: :up, - last_reference: time, - crf: current_crf, conn_state: :active, retries: 0 }) else - :error_gun_info -> - Logger.debug(":gun.info caused error") - state - {false, key} -> - Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_up message for conn which is not found in state") - :ok = Gun.close(conn_pid) state @@ -224,7 +212,6 @@ defmodule Pleroma.Pool.Connections do with {key, conn} <- find_conn(state.conns, conn_pid), {true, key} <- {Process.alive?(conn_pid), key} do if conn.retries == retries do - Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") :ok = Gun.close(conn.conn) put_in( @@ -240,18 +227,13 @@ defmodule Pleroma.Pool.Connections do end else {false, key} -> - # gun can send gun_down for closed conn, maybe connection is not closed yet - Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") - put_in( state.conns, Map.delete(state.conns, key) ) nil -> - Logger.debug(":gun_down message for conn which is not found in state") - - :ok = Gun.close(conn_pid) + Logger.debug(":gun_down for conn which isn't found in state") state end @@ -275,7 +257,7 @@ defmodule Pleroma.Pool.Connections do ) else nil -> - Logger.debug(":DOWN message for conn which is not found in state") + Logger.debug(":DOWN for conn which isn't found in state") state end @@ -283,18 +265,6 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end - defp compose_key_gun_info(pid) do - %{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(origin_host) do - {:error, :einval} -> origin_host - ip -> ip - end - - "#{scheme}:#{host}:#{port}" - end - defp find_conn(conns, conn_pid) do Enum.find(conns, fn {_key, conn} -> conn.conn == conn_pid @@ -310,8 +280,4 @@ defmodule Pleroma.Pool.Connections do def crf(current, steps, crf) do 1 + :math.pow(0.5, current / steps) * crf end - - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do - "#{scheme}://#{host}#{path}" - end end -- cgit v1.2.3 From 98ed0d1c4bd2db354154cc4a1d1e6530eb68f499 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 13 Mar 2020 09:37:57 +0300 Subject: more clean up --- lib/pleroma/http/adapter_helper/gun.ex | 2 +- lib/pleroma/http/adapter_helper/hackney.ex | 2 +- lib/pleroma/http/connection.ex | 12 +++++++----- lib/pleroma/pool/request.ex | 1 - lib/pleroma/pool/supervisor.ex | 17 +++++++---------- lib/pleroma/reverse_proxy/client/tesla.ex | 9 +++++---- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- 7 files changed, 22 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index f14b95c19..ead7cdc6b 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -22,7 +22,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = - Pleroma.Config.get([:http, :proxy_url], nil) + Pleroma.Config.get([:http, :proxy_url]) |> AdapterHelper.format_proxy() config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index d08afae0c..dcb4cac71 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do @spec options(keyword(), URI.t()) :: keyword() def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = Pleroma.Config.get([:http, :proxy_url]) config_opts = Pleroma.Config.get([:http, :adapter], []) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 76de3fcfe..ebacf7902 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -30,12 +30,12 @@ defmodule Pleroma.HTTP.Connection do @defaults |> pool_timeout() |> Keyword.merge(opts) - |> adapter().options(uri) + |> adapter_helper().options(uri) end defp pool_timeout(opts) do {config_key, default} = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if adapter() == Tesla.Adapter.Gun do {:pools, Config.get([:pools, :default, :timeout])} else {:hackney_pools, 10_000} @@ -47,10 +47,12 @@ defmodule Pleroma.HTTP.Connection do end @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter().after_request(opts) + def after_request(opts), do: adapter_helper().after_request(opts) - defp adapter do - case Application.get_env(:tesla, :adapter) do + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do Tesla.Adapter.Gun -> AdapterHelper.Gun Tesla.Adapter.Hackney -> AdapterHelper.Hackney _ -> AdapterHelper diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex index db7c10c01..3fb930db7 100644 --- a/lib/pleroma/pool/request.ex +++ b/lib/pleroma/pool/request.ex @@ -39,7 +39,6 @@ defmodule Pleroma.Pool.Request do @impl true def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - # don't flush messages here, because gun can reconnect {:noreply, state} end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex index 8dc5b64b7..faf646cb2 100644 --- a/lib/pleroma/pool/supervisor.ex +++ b/lib/pleroma/pool/supervisor.ex @@ -13,16 +13,13 @@ defmodule Pleroma.Pool.Supervisor do end def init(_) do - children = - [ - %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - ] ++ pools() - - Supervisor.init(children, strategy: :one_for_one) + conns_child = %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} + } + + Supervisor.init([conns_child | pools()], strategy: :one_for_one) end defp pools do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index dbc6b66a3..e81ea8bde 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() - @behaviour Pleroma.ReverseProxy.Client - @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: {:ok, status(), headers} | {:ok, status(), headers, map()} @@ -18,7 +18,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() - opts = Keyword.merge(opts, body_as: :chunks) + opts = Keyword.put(opts, :body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -39,7 +39,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do end @impl true - @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done + @spec stream_body(map()) :: + {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() def stream_body(%{pid: pid, opts: opts, fin: true}) do # if connection was reused, but in tesla were redirects, # tesla returns new opened connection, which must be closed manually diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 8f1aa3200..35b973b56 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -59,7 +59,7 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [gun](https://github.com/ninenines/gun). + * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun). """ @default_options [pool: :media] -- cgit v1.2.3 From 7c8003c3fcdcab075b9722ab236bf2d1d0e0e8cd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 15 Mar 2020 21:00:12 +0300 Subject: [#1364] Improved control over generation / sending of notifications. Fixed blocking / muting users notifications issue. Added tests. --- lib/pleroma/activity.ex | 10 ++ lib/pleroma/notification.ex | 127 ++++++++++++++++++------- lib/pleroma/thread_mute.ex | 37 +++++-- lib/pleroma/user.ex | 50 ++++++++-- lib/pleroma/user_relationship.ex | 9 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 +- 6 files changed, 185 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6ca05f74e..bbaa561a7 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -95,6 +95,16 @@ defmodule Pleroma.Activity do |> preload([activity, object: object], object: object) end + def user_actor(%Activity{actor: nil}), do: nil + + def user_actor(%Activity{} = activity) do + with %User{} <- activity.user_actor do + activity.user_actor + else + _ -> User.get_cached_by_ap_id(activity.actor) + end + end + def with_joined_user_actor(query, join_type \\ :inner) do join(query, join_type, [activity], u in User, on: u.ap_id == activity.actor, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 60dba3434..0d7a6610a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Notification do alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Push @@ -17,6 +18,7 @@ defmodule Pleroma.Notification do import Ecto.Query import Ecto.Changeset + require Logger @type t :: %__MODULE__{} @@ -101,7 +103,7 @@ defmodule Pleroma.Notification do query |> where([n, a], a.actor not in ^notification_muted_ap_ids) - |> join(:left, [n, a], tm in Pleroma.ThreadMute, + |> join(:left, [n, a], tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) ) |> where([n, a, o, tm], is_nil(tm.user_id)) @@ -284,58 +286,108 @@ defmodule Pleroma.Notification do def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity) - unless object && object.data["type"] == "Answer" do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) - {:ok, notifications} - else + if object && object.data["type"] == "Answer" do {:ok, []} + else + do_create_notifications(activity) end end def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do + do_create_notifications(activity) + end + + def create_notifications(_), do: {:ok, []} + + defp do_create_notifications(%Activity{} = activity) do + {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) + potential_receivers = enabled_receivers ++ disabled_receivers + notifications = - activity - |> get_notified_from_activity() - |> Enum.map(&create_notification(activity, &1)) + Enum.map(potential_receivers, fn user -> + do_send = user in enabled_receivers + create_notification(activity, user, do_send) + end) {:ok, notifications} end - def create_notifications(_), do: {:ok, []} - # TODO move to sql, too. - def create_notification(%Activity{} = activity, %User{} = user) do + def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - ["user", "user:notification"] - |> Streamer.stream(notification) + if do_send do + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end - Push.send(notification) notification end end + @doc """ + Returns a tuple with 2 elements: + {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)} + """ def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do - [] - |> Utils.maybe_notify_to_recipients(activity) - |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) - |> Utils.maybe_notify_followers(activity) - |> Enum.uniq() - |> User.get_users_from_set(local_only) + potential_receiver_ap_ids = + [] + |> Utils.maybe_notify_to_recipients(activity) + |> Utils.maybe_notify_mentioned_recipients(activity) + |> Utils.maybe_notify_subscribers(activity) + |> Utils.maybe_notify_followers(activity) + |> Enum.uniq() + + notification_enabled_ap_ids = + potential_receiver_ap_ids + |> exclude_relation_restricting_ap_ids(activity) + |> exclude_thread_muter_ap_ids(activity) + + potential_receivers = + potential_receiver_ap_ids + |> Enum.uniq() + |> User.get_users_from_set(local_only) + + notification_enabled_users = + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + + {notification_enabled_users, potential_receivers -- notification_enabled_users} + end + + def get_notified_from_activity(_, _local_only), do: {[], []} + + @doc "Filters out AP IDs of users basing on their relationships with activity actor user" + def exclude_relation_restricting_ap_ids([], _activity), do: [] + + def exclude_relation_restricting_ap_ids(ap_ids, %Activity{} = activity) do + relation_restricted_ap_ids = + activity + |> Activity.user_actor() + |> User.incoming_relations_ungrouped_ap_ids([ + :block, + :notification_mute + ]) + + Enum.uniq(ap_ids) -- relation_restricted_ap_ids end - def get_notified_from_activity(_, _local_only), do: [] + @doc "Filters out AP IDs of users who mute activity thread" + def exclude_thread_muter_ap_ids([], _activity), do: [] + + def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do + thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"]) + + Enum.uniq(ap_ids) -- thread_muter_ap_ids + end @spec skip?(Activity.t(), User.t()) :: boolean() - def skip?(activity, user) do + def skip?(%Activity{} = activity, %User{} = user) do [ :self, :followers, @@ -344,18 +396,20 @@ defmodule Pleroma.Notification do :non_follows, :recently_followed ] - |> Enum.any?(&skip?(&1, activity, user)) + |> Enum.find(&skip?(&1, activity, user)) end + def skip?(_, _), do: false + @spec skip?(atom(), Activity.t(), User.t()) :: boolean() - def skip?(:self, activity, user) do + def skip?(:self, %Activity{} = activity, %User{} = user) do activity.data["actor"] == user.ap_id end def skip?( :followers, - activity, - %{notification_settings: %{followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) @@ -364,15 +418,19 @@ defmodule Pleroma.Notification do def skip?( :non_followers, - activity, - %{notification_settings: %{non_followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do + def skip?( + :follows, + %Activity{} = activity, + %User{notification_settings: %{follows: false}} = user + ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) @@ -380,15 +438,16 @@ defmodule Pleroma.Notification do def skip?( :non_follows, - activity, - %{notification_settings: %{non_follows: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_follows: false}} = user ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) !User.following?(user, followed) end - def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do + # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL + def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] Notification.for_user(user) diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index cc815430a..2b4cf02cf 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do alias Pleroma.ThreadMute alias Pleroma.User - require Ecto.Query + import Ecto.Changeset + import Ecto.Query schema "thread_mutes" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) @@ -18,19 +19,43 @@ defmodule Pleroma.ThreadMute do def changeset(mute, params \\ %{}) do mute - |> Ecto.Changeset.cast(params, [:user_id, :context]) - |> Ecto.Changeset.foreign_key_constraint(:user_id) - |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index) + |> cast(params, [:user_id, :context]) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:user_id, name: :unique_index) end def query(user_id, context) do {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) ThreadMute - |> Ecto.Query.where(user_id: ^user_id) - |> Ecto.Query.where(context: ^context) + |> where(user_id: ^user_id) + |> where(context: ^context) end + def muters_query(context) do + ThreadMute + |> join(:inner, [tm], u in assoc(tm, :user)) + |> where([tm], tm.context == ^context) + |> select([tm, u], u.ap_id) + end + + def muter_ap_ids(context, ap_ids \\ nil) + + def muter_ap_ids(context, ap_ids) when context not in [nil, ""] do + context + |> muters_query() + |> maybe_filter_on_ap_id(ap_ids) + |> Repo.all() + end + + def muter_ap_ids(_context, _ap_ids), do: [] + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [tm, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def add_mute(user_id, context) do %ThreadMute{} |> changeset(%{user_id: user_id, context: context}) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index db510d957..8c8ecfe35 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -149,22 +149,26 @@ defmodule Pleroma.User do {outgoing_relation, outgoing_relation_target}, {incoming_relation, incoming_relation_source} ]} <- @user_relationships_config do - # Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc. + # Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes, + # :notification_muter_mutes, :subscribee_subscriptions has_many(outgoing_relation, UserRelationship, foreign_key: :source_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc. + # Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes, + # :notification_mutee_mutes, :subscriber_subscriptions has_many(incoming_relation, UserRelationship, foreign_key: :target_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blocked_users`, `has_many :muted_users` etc. + # Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users, + # :notification_muted_users, :subscriber_users has_many(outgoing_relation_target, through: [outgoing_relation, :target]) - # Definitions of `has_many :blocker_users`, `has_many :muter_users` etc. + # Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users, + # :notification_muter_users, :subscribee_users has_many(incoming_relation_source, through: [incoming_relation, :source]) end @@ -184,7 +188,9 @@ defmodule Pleroma.User do for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <- @user_relationships_config do - # Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc. + # `def blocked_users_relation/2`, `def muted_users_relation/2`, + # `def reblog_muted_users_relation/2`, `def notification_muted_users/2`, + # `def subscriber_users/2` def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do target_users_query = assoc(user, unquote(outgoing_relation_target)) @@ -195,7 +201,8 @@ defmodule Pleroma.User do end end - # Definitions of `blocked_users/1`, `muted_users/1`, etc. + # `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`, + # `def notification_muted_users/2`, `def subscriber_users/2` def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -205,7 +212,8 @@ defmodule Pleroma.User do |> Repo.all() end - # Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc. + # `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`, + # `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2` def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -1217,7 +1225,9 @@ defmodule Pleroma.User do E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` """ @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} - def outgoing_relations_ap_ids(_, []), do: %{} + def outgoing_relations_ap_ids(_user, []), do: %{} + + def outgoing_relations_ap_ids(nil, _relationship_types), do: %{} def outgoing_relations_ap_ids(%User{} = user, relationship_types) when is_list(relationship_types) do @@ -1238,6 +1248,30 @@ defmodule Pleroma.User do ) end + def incoming_relations_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) + + def incoming_relations_ungrouped_ap_ids(_user, [], _ap_ids), do: [] + + def incoming_relations_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] + + def incoming_relations_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) + when is_list(relationship_types) do + user + |> assoc(:incoming_relationships) + |> join(:inner, [user_rel], u in assoc(user_rel, :source)) + |> where([user_rel, u], user_rel.relationship_type in ^relationship_types) + |> maybe_filter_on_ap_id(ap_ids) + |> select([user_rel, u], u.ap_id) + |> distinct(true) + |> Repo.all() + end + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [user_rel, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def deactivate_async(user, status \\ true) do BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 393947942..01b6ace9d 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -21,15 +21,18 @@ defmodule Pleroma.UserRelationship do end for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do - # Definitions of `create_block/2`, `create_mute/2` etc. + # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`, + # `def create_notification_mute/2`, `def create_inverse_subscription/2` def unquote(:"create_#{relationship_type}")(source, target), do: create(unquote(relationship_type), source, target) - # Definitions of `delete_block/2`, `delete_mute/2` etc. + # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`, + # `def delete_notification_mute/2`, `def delete_inverse_subscription/2` def unquote(:"delete_#{relationship_type}")(source, target), do: delete(unquote(relationship_type), source, target) - # Definitions of `block_exists?/2`, `mute_exists?/2` etc. + # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`, + # `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2` def unquote(:"#{relationship_type}_exists?")(source, target), do: exists?(unquote(relationship_type), source, target) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..d6549a932 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1108,13 +1108,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def add_mention_tags(object) do - mentions = - object - |> Utils.get_notified_from_object() - |> Enum.map(&build_mention_tag/1) + {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object) + potential_receivers = enabled_receivers ++ disabled_receivers + mentions = Enum.map(potential_receivers, &build_mention_tag/1) tags = object["tag"] || [] - Map.put(object, "tag", tags ++ mentions) end -- cgit v1.2.3 From 35471205f862fa069c6d87aefc1d827c9fab6e08 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 15:47:25 +0300 Subject: temp fix for `:gun.info` MatchError --- lib/pleroma/pool/connections.ex | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 772833509..16aa80548 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -169,19 +169,26 @@ defmodule Pleroma.Pool.Connections do @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip + # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 + # TODO: REMOVE LATER + {key, conn} = + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + find_conn(state.conns, conn_pid, key) + rescue + MatcheError -> find_conn(state.conns, conn_pid) end - key = "#{scheme}:#{host}:#{port}" - state = - with {_key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do + with {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From bf474ca3c154544b54720ea23c06191e68f32522 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 16 Mar 2020 16:23:49 +0300 Subject: fix --- lib/pleroma/pool/connections.ex | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 16aa80548..91102faf7 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,28 +167,30 @@ defmodule Pleroma.Pool.Connections do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do + defp find_conn_from_gun_info(conns, pid) do # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 # TODO: REMOVE LATER - {key, conn} = - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - find_conn(state.conns, conn_pid, key) - rescue - MatcheError -> find_conn(state.conns, conn_pid) - end + try do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + find_conn(conns, pid, key) + rescue + MatcheError -> find_conn(conns, pid) + end + end + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do state = - with {true, key} <- {Process.alive?(conn_pid), key} do + with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn | gun_state: :up, -- cgit v1.2.3 From d198e7fa2a0c92be4e99c5a765de85096d318bfe Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 28 Jan 2020 09:47:59 +0300 Subject: Admin API: `PATCH /api/pleroma/admin/users/:nickname/change_password` --- lib/pleroma/moderation_log.ex | 11 ++++++++ lib/pleroma/web/admin_api/admin_api_controller.ex | 33 +++++++++++++++++++++++ lib/pleroma/web/router.ex | 1 + 3 files changed, 45 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index e32895f70..b5435a553 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -605,6 +605,17 @@ defmodule Pleroma.ModerationLog do }" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "change_password", + "subject" => subjects + } + }) do + "@#{actor_nickname} changed password for users: #{users_to_nicknames_string(subjects)}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 175260bc2..2aa2c6ac2 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -658,6 +658,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json_response(conn, :no_content, "") end + @doc "Changes password for a given user" + def change_password(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do + with {_, user} <- {:user, User.get_cached_by_nickname(nickname)}, + {:ok, _user} <- + User.reset_password(user, %{ + password: params["new_password"], + password_confirmation: params["new_password"] + }) do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "change_password" + }) + + User.force_password_reset_async(user) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "force_password_reset" + }) + + json(conn, %{status: "success"}) + else + {:error, changeset} -> + {_, {error, _}} = Enum.at(changeset.errors, 0) + json(conn, %{error: "New password #{error}."}) + + _ -> + json(conn, %{error: "Unable to change password."}) + end + end + def list_reports(conn, params) do {page, page_size} = page_params(params) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e4e3ee704..c03ad101e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -173,6 +173,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) patch("/users/force_password_reset", AdminAPIController, :force_password_reset) + patch("/users/:nickname/change_password", AdminAPIController, :change_password) get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) -- cgit v1.2.3 From 13cce9c0debbf9a80ed5da26cb34ca563e5e1417 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Fri, 31 Jan 2020 21:07:46 +0300 Subject: Admin API: `PATCH /api/pleroma/admin/users/:nickname/credentials`, `GET /api/pleroma/admin/users/:nickname/credentials`. --- lib/pleroma/moderation_log.ex | 4 +- lib/pleroma/user.ex | 86 +++++++++++++++++++++- lib/pleroma/web/admin_api/admin_api_controller.ex | 34 ++++++--- lib/pleroma/web/admin_api/views/account_view.ex | 40 ++++++++++ .../mastodon_api/controllers/account_controller.ex | 60 +++------------ lib/pleroma/web/router.ex | 3 +- 6 files changed, 163 insertions(+), 64 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index b5435a553..7aacd9d80 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -609,11 +609,11 @@ defmodule Pleroma.ModerationLog do def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, - "action" => "change_password", + "action" => "updated_users", "subject" => subjects } }) do - "@#{actor_nickname} changed password for users: #{users_to_nicknames_string(subjects)}" + "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}" end defp nicknames_to_string(nicknames) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 911dde6e2..44de64345 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -417,9 +417,55 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> put_fields() + |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) + |> put_change_if_present(:avatar, &put_upload(&1, :avatar)) + |> put_change_if_present(:banner, &put_upload(&1, :banner)) + |> put_change_if_present(:background, &put_upload(&1, :background)) + |> put_change_if_present( + :pleroma_settings_store, + &{:ok, Map.merge(struct.pleroma_settings_store, &1)} + ) |> validate_fields(false) end + defp put_fields(changeset) do + if raw_fields = get_change(changeset, :raw_fields) do + raw_fields = + raw_fields + |> Enum.filter(fn %{"name" => n} -> n != "" end) + + fields = + raw_fields + |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + + changeset + |> put_change(:raw_fields, raw_fields) + |> put_change(:fields, fields) + else + changeset + end + end + + defp put_change_if_present(changeset, map_field, value_function) do + if value = get_change(changeset, map_field) do + with {:ok, new_value} <- value_function.(value) do + put_change(changeset, map_field, new_value) + else + _ -> changeset + end + else + changeset + end + end + + defp put_upload(value, type) do + with %Plug.Upload{} <- value, + {:ok, object} <- ActivityPub.upload(value, type: type) do + {:ok, object.data} + end + end + def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) @@ -463,6 +509,27 @@ defmodule Pleroma.User do |> validate_fields(remote?) end + def update_as_admin_changeset(struct, params) do + struct + |> update_changeset(params) + |> cast(params, [:email]) + |> delete_change(:also_known_as) + |> unique_constraint(:email) + |> validate_format(:email, @email_regex) + end + + @spec update_as_admin(%User{}, map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} + def update_as_admin(user, params) do + params = Map.put(params, "password_confirmation", params["password"]) + changeset = update_as_admin_changeset(user, params) + + if params["password"] do + reset_password(user, changeset, params) + else + User.update_and_set_cache(changeset) + end + end + def password_update_changeset(struct, params) do struct |> cast(params, [:password, :password_confirmation]) @@ -473,10 +540,14 @@ defmodule Pleroma.User do end @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} - def reset_password(%User{id: user_id} = user, data) do + def reset_password(%User{} = user, params) do + reset_password(user, user, params) + end + + def reset_password(%User{id: user_id} = user, struct, params) do multi = Multi.new() - |> Multi.update(:user, password_update_changeset(user, data)) + |> Multi.update(:user, password_update_changeset(struct, params)) |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id)) |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user)) @@ -1856,6 +1927,17 @@ defmodule Pleroma.User do def fields(%{fields: fields}), do: fields + def sanitized_fields(%User{} = user) do + user + |> User.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => name, + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) + end + def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Pleroma.Config.get([:instance, limit_name], 0) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2aa2c6ac2..0368df1e9 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get] + when action in [:list_users, :user_show, :right_get, :show_user_credentials] ) plug( @@ -54,7 +54,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :tag_users, :untag_users, :right_add, - :right_delete + :right_delete, + :update_user_credentials ] ) @@ -658,21 +659,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json_response(conn, :no_content, "") end - @doc "Changes password for a given user" - def change_password(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do + @doc "Show a given user's credentials" + def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + conn + |> put_view(AccountView) + |> render("credentials.json", %{user: user, for: admin}) + else + _ -> {:error, :not_found} + end + end + + @doc "Updates a given user" + def update_user_credentials( + %{assigns: %{user: admin}} = conn, + %{"nickname" => nickname} = params + ) do with {_, user} <- {:user, User.get_cached_by_nickname(nickname)}, {:ok, _user} <- - User.reset_password(user, %{ - password: params["new_password"], - password_confirmation: params["new_password"] - }) do + User.update_as_admin(user, params) do ModerationLog.insert_log(%{ actor: admin, subject: [user], - action: "change_password" + action: "updated_users" }) - User.force_password_reset_async(user) + if params["password"] do + User.force_password_reset_async(user) + end ModerationLog.insert_log(%{ actor: admin, diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 1e03849de..a16a3ebf0 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -23,6 +23,43 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + def render("credentials.json", %{user: user, for: for_user}) do + user = User.sanitize_html(user, User.html_filter_policy(for_user)) + avatar = User.avatar_url(user) |> MediaProxy.url() + banner = User.banner_url(user) |> MediaProxy.url() + background = image_url(user.background) |> MediaProxy.url() + + user + |> Map.take([ + :id, + :bio, + :email, + :fields, + :name, + :nickname, + :locked, + :no_rich_text, + :default_scope, + :hide_follows, + :hide_followers_count, + :hide_follows_count, + :hide_followers, + :hide_favorites, + :allow_following_move, + :show_role, + :skip_thread_containment, + :pleroma_settings_store, + :raw_fields, + :discoverable, + :actor_type + ]) + |> Map.merge(%{ + "avatar" => avatar, + "banner" => banner, + "background" => background + }) + end + def render("show.json", %{user: user}) do avatar = User.avatar_url(user) |> MediaProxy.url() display_name = Pleroma.HTML.strip_tags(user.name || user.nickname) @@ -104,4 +141,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do "" end end + + defp image_url(%{"url" => [%{"href" => href} | _]}), do: href + defp image_url(_), do: nil end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 88c997b9f..56e6214c5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, truthy_param?: 1, assign_account_by_id: 2, json_response: 3] - alias Pleroma.Emoji alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter alias Pleroma.User @@ -140,17 +139,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do def update_credentials(%{assigns: %{user: original_user}} = conn, params) do user = original_user - params = - if Map.has_key?(params, "fields_attributes") do - Map.update!(params, "fields_attributes", fn fields -> - fields - |> normalize_fields_attributes() - |> Enum.filter(fn %{"name" => n} -> n != "" end) - end) - else - params - end - user_params = [ :no_rich_text, @@ -169,46 +157,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do add_if_present(acc, params, to_string(key), key, &{:ok, truthy_param?(&1)}) end) |> add_if_present(params, "display_name", :name) - |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end) - |> add_if_present(params, "avatar", :avatar, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :avatar) do - {:ok, object.data} - end - end) - |> add_if_present(params, "header", :banner, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :banner) do - {:ok, object.data} - end - end) - |> add_if_present(params, "pleroma_background_image", :background, fn value -> - with %Plug.Upload{} <- value, - {:ok, object} <- ActivityPub.upload(value, type: :background) do - {:ok, object.data} - end - end) - |> add_if_present(params, "fields_attributes", :fields, fn fields -> - fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) - - {:ok, fields} - end) - |> add_if_present(params, "fields_attributes", :raw_fields) - |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> - {:ok, Map.merge(user.pleroma_settings_store, value)} - end) + |> add_if_present(params, "note", :bio) + |> add_if_present(params, "avatar", :avatar) + |> add_if_present(params, "header", :banner) + |> add_if_present(params, "pleroma_background_image", :background) + |> add_if_present( + params, + "fields_attributes", + :raw_fields, + &{:ok, normalize_fields_attributes(&1)} + ) + |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store) |> add_if_present(params, "default_scope", :default_scope) |> add_if_present(params, "actor_type", :actor_type) - emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") - - user_emojis = - user - |> Map.get(:emoji, []) - |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text)) - |> Enum.dedup() - - user_params = Map.put(user_params, :emoji, user_emojis) changeset = User.update_changeset(user, user_params) with {:ok, user} <- User.update_and_set_cache(changeset) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c03ad101e..2927775eb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -173,7 +173,8 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset) patch("/users/force_password_reset", AdminAPIController, :force_password_reset) - patch("/users/:nickname/change_password", AdminAPIController, :change_password) + get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials) + patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials) get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) -- cgit v1.2.3 From 74388336852b18d5d5f108a8305f1a038301f7a1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 16 Mar 2020 21:58:10 +0300 Subject: [#1364] Improved notification-related tests. --- lib/pleroma/notification.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 0d7a6610a..104368fd1 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -344,6 +344,7 @@ defmodule Pleroma.Notification do |> Utils.maybe_notify_followers(activity) |> Enum.uniq() + # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs notification_enabled_ap_ids = potential_receiver_ap_ids |> exclude_relation_restricting_ap_ids(activity) -- cgit v1.2.3 From f9d622d25a744f58fbaf8370ad4435597bb15bf0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Mar 2020 15:08:49 +0100 Subject: WIP --- lib/pleroma/web/activity_pub/transmogrifier.ex | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..db848f657 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -202,21 +202,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("conversation", context) end - def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do - attachments = - Enum.map(attachment, fn data -> - media_type = data["mediaType"] || data["mimeType"] - href = data["url"] || data["href"] - url = [%{"type" => "Link", "mediaType" => media_type, "href" => href}] - - data - |> Map.put("mediaType", media_type) - |> Map.put("url", url) - end) - - Map.put(object, "attachment", attachments) - end - def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do object |> Map.put("attachment", [attachment]) -- cgit v1.2.3 From 7d275970ab191af539acbc0baec3bc1d0a2558e1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 19 Mar 2020 10:08:11 -0500 Subject: Add emoji reactions to features in nodeinfo --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 18eb41333..c653a80c3 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -74,7 +74,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do end, if Config.get([:instance, :safe_dm_mentions]) do "safe_dm_mentions" - end + end, + "pleroma_emoji_reactions" ] |> Enum.filter(& &1) -- cgit v1.2.3 From 9b9d67bbec537df6f7c5729e81da6deeaf896bd9 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 19 Mar 2020 18:16:12 +0100 Subject: Fix linting. --- lib/pleroma/web/activity_pub/object_validators/create_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex index bd90f7250..9e480c4ed 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset -- cgit v1.2.3 From c1fd4f665335ba67336bd1b2fab2d9df5e247e08 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Mar 2020 19:10:03 +0100 Subject: transmogrifier.ex: rework fix_attachment for better IR --- lib/pleroma/web/activity_pub/transmogrifier.ex | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index db848f657..df5ca0239 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -202,6 +202,51 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("conversation", context) end + defp add_if_present(map, _key, nil), do: map + + defp add_if_present(map, key, value) do + Map.put(map, key, value) + end + + def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do + attachments = + Enum.map(attachment, fn data -> + url = + cond do + is_list(data["url"]) -> List.first(data["url"]) + is_map(data["url"]) -> data["url"] + true -> nil + end + + media_type = + cond do + is_map(url) && is_binary(url["mediaType"]) -> url["mediaType"] + is_binary(data["mediaType"]) -> data["mediaType"] + is_binary(data["mimeType"]) -> data["mimeType"] + true -> nil + end + + href = + cond do + is_map(url) && is_binary(url["href"]) -> url["href"] + is_binary(data["url"]) -> data["url"] + is_binary(data["href"]) -> data["href"] + end + + attachment_url = + %{"href" => href} + |> add_if_present("mediaType", media_type) + |> add_if_present("type", Map.get(url || %{}, "type")) + + %{"url" => [attachment_url]} + |> add_if_present("mediaType", media_type) + |> add_if_present("type", data["type"]) + |> add_if_present("name", data["name"]) + end) + + Map.put(object, "attachment", attachments) + end + def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do object |> Map.put("attachment", [attachment]) -- cgit v1.2.3 From c2e415143b1dfe5d89eff06fbce6840c445aa5fa Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 22 Mar 2020 21:51:44 +0300 Subject: WIP: preloading of user relations for timeline/statuses rendering (performance improvement). --- lib/pleroma/user.ex | 6 +- lib/pleroma/user_relationship.ex | 44 ++++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 69 ++++++++++++++++++---- lib/pleroma/web/mastodon_api/views/status_view.ex | 58 ++++++++++++++++-- 4 files changed, 159 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 12c2ad815..daaa6d86b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1642,8 +1642,12 @@ defmodule Pleroma.User do |> Repo.all() end + def muting_reblogs?(%User{} = user, %User{} = target) do + UserRelationship.reblog_mute_exists?(user, target) + end + def showing_reblogs?(%User{} = user, %User{} = target) do - not UserRelationship.reblog_mute_exists?(user, target) + not muting_reblogs?(user, target) end @doc """ diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 393947942..167a3919c 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias FlakeId.Ecto.CompatType alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -34,6 +35,10 @@ defmodule Pleroma.UserRelationship do do: exists?(unquote(relationship_type), source, target) end + def user_relationship_types, do: Keyword.keys(user_relationship_mappings()) + + def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__() + def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do user_relationship |> cast(params, [:relationship_type, :source_id, :target_id]) @@ -72,6 +77,45 @@ defmodule Pleroma.UserRelationship do end end + def dictionary( + source_users, + target_users, + source_to_target_rel_types \\ nil, + target_to_source_rel_types \\ nil + ) + when is_list(source_users) and is_list(target_users) do + get_bin_ids = fn user -> + with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id + end + + source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) + target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + + get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end + + source_to_target_rel_types = + Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + target_to_source_rel_types = + Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + __MODULE__ + |> where( + fragment( + "(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \ + (source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^source_to_target_rel_types, + ^target_user_ids, + ^source_user_ids, + ^target_to_source_rel_types + ) + ) + |> select([ur], [ur.relationship_type, ur.source_id, ur.target_id]) + |> Repo.all() + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 4ebce73b4..15a579278 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -10,6 +10,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy + def test_rel(user_relationships, rel_type, source, target, func) do + cond do + is_nil(source) or is_nil(target) -> + false + + user_relationships -> + [rel_type, source.id, target.id] in user_relationships + + true -> + func.(source, target) + end + end + def render("index.json", %{users: users} = opts) do users |> render_many(AccountView, "show.json", opts) @@ -35,21 +48,50 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do - follow_state = User.get_follow_state(user, target) + def render( + "relationship.json", + %{user: %User{} = reading_user, target: %User{} = target} = opts + ) do + user_relationships = Map.get(opts, :user_relationships) + + follow_state = User.get_follow_state(reading_user, target) + # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations %{ id: to_string(target.id), following: follow_state == "accept", - followed_by: User.following?(target, user), - blocking: User.blocks_user?(user, target), - blocked_by: User.blocks_user?(target, user), - muting: User.mutes?(user, target), - muting_notifications: User.muted_notifications?(user, target), - subscribing: User.subscribed_to?(user, target), + followed_by: User.following?(target, reading_user), + blocking: + test_rel(user_relationships, :block, reading_user, target, &User.blocks_user?(&1, &2)), + blocked_by: + test_rel(user_relationships, :block, target, reading_user, &User.blocks_user?(&1, &2)), + muting: test_rel(user_relationships, :mute, reading_user, target, &User.mutes?(&1, &2)), + muting_notifications: + test_rel( + user_relationships, + :notification_mute, + reading_user, + target, + &User.muted_notifications?(&1, &2) + ), + subscribing: + test_rel( + user_relationships, + :inverse_subscription, + target, + reading_user, + &User.subscribed_to?(&2, &1) + ), requested: follow_state == "pending", - domain_blocking: User.blocks_domain?(user, target), - showing_reblogs: User.showing_reblogs?(user, target), + domain_blocking: User.blocks_domain?(reading_user, target), + showing_reblogs: + not test_rel( + user_relationships, + :reblog_mute, + reading_user, + target, + &User.muting_reblogs?(&1, &2) + ), endorsed: false } end @@ -93,7 +135,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - relationship = render("relationship.json", %{user: opts[:for], target: user}) + relationship = + render("relationship.json", %{ + user: opts[:for], + target: user, + user_relationships: opts[:user_relationships] + }) %{ id: to_string(user.id), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f7469cdff..e0c368ec9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -70,11 +71,34 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end + defp user_relationships_opt(opts) do + reading_user = opts[:for] + + if reading_user do + activities = opts[:activities] + actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) + + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + else + [] + end + end + def render("index.json", opts) do - replied_to_activities = get_replied_to_activities(opts.activities) - opts = Map.put(opts, :replied_to_activities, replied_to_activities) + activities = opts.activities + replied_to_activities = get_replied_to_activities(activities) + + opts = + opts + |> Map.put(:replied_to_activities, replied_to_activities) + |> Map.put(:user_relationships, user_relationships_opt(opts)) - safe_render_many(opts.activities, StatusView, "show.json", opts) + safe_render_many(activities, StatusView, "show.json", opts) end def render( @@ -107,7 +131,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: activity_object.data["id"], url: activity_object.data["id"], - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + user_relationships: opts[:user_relationships] + }), in_reply_to_id: nil, in_reply_to_account_id: nil, reblog: reblogged, @@ -253,11 +282,28 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> [] end + user_relationships_opt = opts[:user_relationships] + + muted = + thread_muted? || + Pleroma.Web.MastodonAPI.AccountView.test_rel( + user_relationships_opt, + :mute, + opts[:for], + user, + fn for_user, user -> User.mutes?(for_user, user) end + ) + %{ id: to_string(activity.id), uri: object.data["id"], url: url, - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + user_relationships: user_relationships_opt + }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, @@ -270,7 +316,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: thread_muted? || User.mutes?(opts[:for], user), + muted: muted, pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary, -- cgit v1.2.3 From 3c78e5f3275494b3dc4546e65f19eb3a3c97033a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 23 Mar 2020 12:01:11 +0300 Subject: Preloading of follow relations for timeline/statuses rendering (performance improvement). Refactoring. --- lib/pleroma/following_relationship.ex | 26 ++++++++ lib/pleroma/user.ex | 7 ++ lib/pleroma/user_relationship.ex | 13 ++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 75 ++++++++++++++++------ lib/pleroma/web/mastodon_api/views/status_view.ex | 46 ++++++++----- 5 files changed, 130 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index a6d281151..dd1696136 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -129,4 +129,30 @@ defmodule Pleroma.FollowingRelationship do move_following(origin, target) end end + + def all_between_user_sets( + source_users, + target_users + ) + when is_list(source_users) and is_list(target_users) do + get_bin_ids = fn user -> + with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id + end + + source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) + target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + + __MODULE__ + |> where( + fragment( + "(follower_id = ANY(?) AND following_id = ANY(?)) OR \ + (follower_id = ANY(?) AND following_id = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^target_user_ids, + ^source_user_ids + ) + ) + |> Repo.all() + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index daaa6d86b..eb72755a0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -674,7 +674,14 @@ defmodule Pleroma.User do def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) + get_follow_state(follower, following, following_relationship) + end + def get_follow_state( + %User{} = follower, + %User{} = following, + following_relationship + ) do case {following_relationship, following.local} do {nil, false} -> case Utils.fetch_latest_follow(follower, following) do diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 167a3919c..9423e3a42 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -116,6 +116,19 @@ defmodule Pleroma.UserRelationship do |> Repo.all() end + def exists?(dictionary, rel_type, source, target, func) do + cond do + is_nil(source) or is_nil(target) -> + false + + dictionary -> + [rel_type, source.id, target.id] in dictionary + + true -> + func.(source, target) + end + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 15a579278..2fe46158b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -6,21 +6,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do use Pleroma.Web, :view alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy - def test_rel(user_relationships, rel_type, source, target, func) do - cond do - is_nil(source) or is_nil(target) -> - false - - user_relationships -> - [rel_type, source.id, target.id] in user_relationships - - true -> - func.(source, target) - end + defp find_following_rel(following_relationships, follower, following) do + Enum.find(following_relationships, fn + fr -> fr.follower_id == follower.id and fr.following_id == following.id + end) end def render("index.json", %{users: users} = opts) do @@ -53,21 +47,61 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{user: %User{} = reading_user, target: %User{} = target} = opts ) do user_relationships = Map.get(opts, :user_relationships) + following_relationships = opts[:following_relationships] + + follow_state = + if following_relationships do + user_to_target_following_relation = + find_following_rel(following_relationships, reading_user, target) + + User.get_follow_state(reading_user, target, user_to_target_following_relation) + else + User.get_follow_state(reading_user, target) + end - follow_state = User.get_follow_state(reading_user, target) + followed_by = + if following_relationships do + with %{state: "accept"} <- + find_following_rel(following_relationships, target, reading_user) do + true + else + _ -> false + end + else + User.following?(target, reading_user) + end # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations %{ id: to_string(target.id), following: follow_state == "accept", - followed_by: User.following?(target, reading_user), + followed_by: followed_by, blocking: - test_rel(user_relationships, :block, reading_user, target, &User.blocks_user?(&1, &2)), + UserRelationship.exists?( + user_relationships, + :block, + reading_user, + target, + &User.blocks_user?(&1, &2) + ), blocked_by: - test_rel(user_relationships, :block, target, reading_user, &User.blocks_user?(&1, &2)), - muting: test_rel(user_relationships, :mute, reading_user, target, &User.mutes?(&1, &2)), + UserRelationship.exists?( + user_relationships, + :block, + target, + reading_user, + &User.blocks_user?(&1, &2) + ), + muting: + UserRelationship.exists?( + user_relationships, + :mute, + reading_user, + target, + &User.mutes?(&1, &2) + ), muting_notifications: - test_rel( + UserRelationship.exists?( user_relationships, :notification_mute, reading_user, @@ -75,7 +109,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do &User.muted_notifications?(&1, &2) ), subscribing: - test_rel( + UserRelationship.exists?( user_relationships, :inverse_subscription, target, @@ -85,7 +119,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do requested: follow_state == "pending", domain_blocking: User.blocks_domain?(reading_user, target), showing_reblogs: - not test_rel( + not UserRelationship.exists?( user_relationships, :reblog_mute, reading_user, @@ -139,7 +173,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render("relationship.json", %{ user: opts[:for], target: user, - user_relationships: opts[:user_relationships] + user_relationships: opts[:user_relationships], + following_relationships: opts[:following_relationships] }) %{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e0c368ec9..55a5513f9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.ActivityExpiration + alias Pleroma.FollowingRelationship alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -71,22 +72,31 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - defp user_relationships_opt(opts) do + defp relationships_opts(opts) do reading_user = opts[:for] - if reading_user do - activities = opts[:activities] - actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) + {user_relationships, following_relationships} = + if reading_user do + activities = opts[:activities] + actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - else - [] - end + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = + FollowingRelationship.all_between_user_sets([reading_user], actors) + + {user_relationships, following_relationships} + else + {[], []} + end + + %{user_relationships: user_relationships, following_relationships: following_relationships} end def render("index.json", opts) do @@ -96,7 +106,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do opts = opts |> Map.put(:replied_to_activities, replied_to_activities) - |> Map.put(:user_relationships, user_relationships_opt(opts)) + |> Map.merge(relationships_opts(opts)) safe_render_many(activities, StatusView, "show.json", opts) end @@ -135,7 +145,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: opts[:user_relationships] + user_relationships: opts[:user_relationships], + following_relationships: opts[:following_relationships] }), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -286,7 +297,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do muted = thread_muted? || - Pleroma.Web.MastodonAPI.AccountView.test_rel( + UserRelationship.exists?( user_relationships_opt, :mute, opts[:for], @@ -302,7 +313,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: user_relationships_opt + user_relationships: user_relationships_opt, + following_relationships: opts[:following_relationships] }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), -- cgit v1.2.3 From 3bd2829e5c125f961b7508bf40ef534a21070562 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 23 Mar 2020 18:56:01 +0100 Subject: Benchmarks: Add timeline benchmark --- lib/pleroma/web/controller_helper.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index ad293cda9..b49523ec3 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -34,7 +34,12 @@ defmodule Pleroma.Web.ControllerHelper do defp param_to_integer(_, default), do: default - def add_link_headers(conn, activities, extra_params \\ %{}) do + def add_link_headers(conn, activities, extra_params \\ %{}) + + def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params), + do: conn + + def add_link_headers(conn, activities, extra_params) do case List.last(activities) do %{id: max_id} -> params = -- cgit v1.2.3 From d1a9716a988fe9f670033ad46cc9637038fbd1e8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Mar 2020 17:38:18 +0400 Subject: Fix activity deletion --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 30e282840..974231925 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -583,6 +583,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + defp do_delete(%Object{data: %{"type" => "Tombstone", "id" => ap_id}}, _) do + activity = + ap_id + |> Activity.Queries.by_object_id() + |> Activity.Queries.by_type("Delete") + |> Repo.one() + + {:ok, activity} + end + @spec block(User.t(), User.t(), String.t() | nil, boolean()) :: {:ok, Activity.t()} | {:error, any()} def block(blocker, blocked, activity_id \\ nil, local \\ true) do -- cgit v1.2.3 From 74560e888e5e3e4dc2fa5b4fec4cf3986a1d1a55 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 24 Mar 2020 18:20:58 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/object_validators/create_validator.ex --- lib/pleroma/web/activity_pub/object_validators/create_validator.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex index 9e480c4ed..872a12c48 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -25,7 +25,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do end def cast_data(data) do - %__MODULE__{} - |> cast(data, __schema__(:fields)) + cast(%__MODULE__{}, data, __schema__(:fields)) end end -- cgit v1.2.3 From aaf00f1ff59fc279758f5fa5ceaf758d683bd216 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 24 Mar 2020 18:24:09 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/pipeline.ex --- lib/pleroma/web/activity_pub/pipeline.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index cb3571917..25f29bf63 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do {:ok, :not_federated} end else - _e -> {:error, "local not set in meta"} + _e -> {:error, :badarg} end end end -- cgit v1.2.3 From f31688246470273cc35588d0f1c2187edc6084c7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 24 Mar 2020 18:37:53 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/activity_pub.ex --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d9f30e629..dd4b04185 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -150,7 +150,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), - # ??? {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, {:containment, :ok} <- {:containment, Containment.contain_child(map)}, {:ok, map, object} <- insert_full_object(map) do -- cgit v1.2.3 From 13cbb9f6ada8dcb15bb7ed12be4d88a18c5db7f7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 24 Mar 2020 22:14:26 +0300 Subject: Implemented preloading of relationships with parent activities' actors for statuses/timeline rendering. Applied preloading for notifications rendering. Fixed announces rendering issue (preloading-related). --- lib/pleroma/activity/queries.ex | 7 ++ lib/pleroma/web/mastodon_api/views/account_view.ex | 15 ++-- .../web/mastodon_api/views/notification_view.ex | 98 +++++++++++++++++----- lib/pleroma/web/mastodon_api/views/status_view.ex | 89 +++++++++++--------- 4 files changed, 140 insertions(+), 69 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 04593b9fb..a34c20343 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Activity.Queries do from(a in query, where: a.actor == ^ap_id) end + def find_by_object_ap_id(activities, object_ap_id) do + Enum.find( + activities, + &(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]]) + ) + end + @spec by_object_id(query, String.t() | [String.t()]) :: query def by_object_id(query \\ Activity, object_id) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2fe46158b..89bea9957 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -46,8 +46,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do "relationship.json", %{user: %User{} = reading_user, target: %User{} = target} = opts ) do - user_relationships = Map.get(opts, :user_relationships) - following_relationships = opts[:following_relationships] + user_relationships = get_in(opts, [:relationships, :user_relationships]) + following_relationships = get_in(opts, [:relationships, :following_relationships]) follow_state = if following_relationships do @@ -61,17 +61,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followed_by = if following_relationships do - with %{state: "accept"} <- - find_following_rel(following_relationships, target, reading_user) do - true - else + case find_following_rel(following_relationships, target, reading_user) do + %{state: "accept"} -> true _ -> false end else User.following?(target, reading_user) end - # TODO: add a note on adjusting StatusView.user_relationships_opt/1 re: preloading of user relations + # NOTE: adjust StatusView.relationships_opts/2 if adding new relation-related flags %{ id: to_string(target.id), following: follow_state == "accept", @@ -173,8 +171,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render("relationship.json", %{ user: opts[:for], target: user, - user_relationships: opts[:user_relationships], - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }) %{ diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 33145c484..e9c618496 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -13,19 +13,68 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - def render("index.json", %{notifications: notifications, for: user}) do - safe_render_many(notifications, NotificationView, "show.json", %{for: user}) + def render("index.json", %{notifications: notifications, for: reading_user}) do + activities = Enum.map(notifications, & &1.activity) + + parent_activities = + activities + |> Enum.filter( + &(Activity.mastodon_notification_type(&1) in [ + "favourite", + "reblog", + "pleroma:emoji_reaction" + ]) + ) + |> Enum.map(& &1.data["object"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Pleroma.Repo.all() + + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + opts = %{ + for: reading_user, + parent_activities: parent_activities, + relationships: StatusView.relationships_opts(reading_user, actors) + } + + safe_render_many(notifications, NotificationView, "show.json", opts) end - def render("show.json", %{ - notification: %Notification{activity: activity} = notification, - for: user - }) do + def render( + "show.json", + %{ + notification: %Notification{activity: activity} = notification, + for: reading_user + } = opts + ) do actor = User.get_cached_by_ap_id(activity.data["actor"]) - parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + + parent_activity_fn = fn -> + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + else + Activity.get_create_by_object_ap_id(activity.data["object"]) + end + end + mastodon_type = Activity.mastodon_notification_type(activity) - with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do + with %{id: _} = account <- + AccountView.render("show.json", %{ + user: actor, + for: reading_user, + relationships: opts[:relationships] + }) do response = %{ id: to_string(notification.id), type: mastodon_type, @@ -36,24 +85,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } + relationships_opts = %{relationships: opts[:relationships]} + case mastodon_type do "mention" -> - put_status(response, activity, user) + put_status(response, activity, reading_user, relationships_opts) "favourite" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, relationships_opts) "reblog" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, relationships_opts) "move" -> - put_target(response, activity, user) + put_target(response, activity, reading_user, relationships_opts) "follow" -> response "pleroma:emoji_reaction" -> - put_status(response, parent_activity, user) |> put_emoji(activity) + response + |> put_status(parent_activity_fn.(), reading_user, relationships_opts) + |> put_emoji(activity) _ -> nil @@ -64,16 +117,21 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do end defp put_emoji(response, activity) do - response - |> Map.put(:emoji, activity.data["content"]) + Map.put(response, :emoji, activity.data["content"]) end - defp put_status(response, activity, user) do - Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user})) + defp put_status(response, activity, reading_user, opts) do + status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user}) + status_render = StatusView.render("show.json", status_render_opts) + + Map.put(response, :status, status_render) end - defp put_target(response, activity, user) do - target = User.get_cached_by_ap_id(activity.data["target"]) - Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user})) + defp put_target(response, activity, reading_user, opts) do + target_user = User.get_cached_by_ap_id(activity.data["target"]) + target_render_opts = Map.merge(opts, %{user: target_user, for: reading_user}) + target_render = AccountView.render("show.json", target_render_opts) + + Map.put(response, :target, target_render) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 55a5513f9..0ef65b352 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -72,41 +72,46 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - defp relationships_opts(opts) do - reading_user = opts[:for] - - {user_relationships, following_relationships} = - if reading_user do - activities = opts[:activities] - actors = Enum.map(activities, fn a -> get_user(a.data["actor"]) end) - - user_relationships = - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - - following_relationships = - FollowingRelationship.all_between_user_sets([reading_user], actors) - - {user_relationships, following_relationships} - else - {[], []} - end + def relationships_opts(_reading_user = nil, _actors) do + %{user_relationships: [], following_relationships: []} + end + + def relationships_opts(reading_user, actors) do + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) %{user_relationships: user_relationships, following_relationships: following_relationships} end def render("index.json", opts) do - activities = opts.activities + # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list + activities = Enum.filter(opts.activities, & &1) replied_to_activities = get_replied_to_activities(activities) + parent_activities = + activities + |> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"])) + |> Enum.map(&Object.normalize(&1).data["id"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) + |> Repo.all() + + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + opts = opts |> Map.put(:replied_to_activities, replied_to_activities) - |> Map.merge(relationships_opts(opts)) + |> Map.put(:parent_activities, parent_activities) + |> Map.put(:relationships, relationships_opts(opts[:for], actors)) safe_render_many(activities, StatusView, "show.json", opts) end @@ -119,17 +124,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do created_at = Utils.to_masto_date(activity.data["published"]) activity_object = Object.normalize(activity) - reblogged_activity = - Activity.create_by_object_ap_id(activity_object.data["id"]) - |> Activity.with_preloaded_bookmark(opts[:for]) - |> Activity.with_set_thread_muted_field(opts[:for]) - |> Repo.one() + reblogged_parent_activity = + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id( + opts[:parent_activities], + activity_object.data["id"] + ) + else + Activity.create_by_object_ap_id(activity_object.data["id"]) + |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) + |> Repo.one() + end - reblogged = render("show.json", Map.put(opts, :activity, reblogged_activity)) + reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity) + reblogged = render("show.json", reblog_rendering_opts) favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) - bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil + bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil mentions = activity.recipients @@ -145,8 +158,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: opts[:user_relationships], - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -156,7 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogs_count: 0, replies_count: 0, favourites_count: 0, - reblogged: reblogged?(reblogged_activity, opts[:for]), + reblogged: reblogged?(reblogged_parent_activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), muted: false, @@ -293,12 +305,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> [] end - user_relationships_opt = opts[:user_relationships] - muted = thread_muted? || UserRelationship.exists?( - user_relationships_opt, + get_in(opts, [:relationships, :user_relationships]), :mute, opts[:for], user, @@ -313,8 +323,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do AccountView.render("show.json", %{ user: user, for: opts[:for], - user_relationships: user_relationships_opt, - following_relationships: opts[:following_relationships] + relationships: opts[:relationships] }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), -- cgit v1.2.3 From e743c2232970e321c833604b232520587ad8e402 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 09:04:00 +0300 Subject: Fixed incorrect usage of "relations" as a short form of "relationships". --- lib/pleroma/notification.ex | 6 +++--- lib/pleroma/user.ex | 20 ++++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++---- .../mastodon_api/controllers/account_controller.ex | 10 +++++++--- lib/pleroma/web/streamer/worker.ex | 2 +- 5 files changed, 25 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 104368fd1..bc691dce3 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -39,11 +39,11 @@ defmodule Pleroma.Notification do end defp for_user_query_ap_id_opts(user, opts) do - ap_id_relations = + ap_id_relationships = [:block] ++ if opts[@include_muted_option], do: [], else: [:notification_mute] - preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships) exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts) @@ -370,7 +370,7 @@ defmodule Pleroma.Notification do relation_restricted_ap_ids = activity |> Activity.user_actor() - |> User.incoming_relations_ungrouped_ap_ids([ + |> User.incoming_relationships_ungrouped_ap_ids([ :block, :notification_mute ]) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 05efc74d4..4919c8e58 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1222,15 +1222,15 @@ defmodule Pleroma.User do end @doc """ - Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type. - E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` + Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type. + E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` """ - @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} - def outgoing_relations_ap_ids(_user, []), do: %{} + @spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} + def outgoing_relationships_ap_ids(_user, []), do: %{} - def outgoing_relations_ap_ids(nil, _relationship_types), do: %{} + def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{} - def outgoing_relations_ap_ids(%User{} = user, relationship_types) + def outgoing_relationships_ap_ids(%User{} = user, relationship_types) when is_list(relationship_types) do db_result = user @@ -1249,13 +1249,13 @@ defmodule Pleroma.User do ) end - def incoming_relations_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) + def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) - def incoming_relations_ungrouped_ap_ids(_user, [], _ap_ids), do: [] + def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: [] - def incoming_relations_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] + def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] - def incoming_relations_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) + def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) when is_list(relationship_types) do user |> assoc(:incoming_relationships) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d9f74b6a4..60e74758f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1230,17 +1230,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp fetch_activities_query_ap_ids_ops(opts) do source_user = opts["muting_user"] - ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: [] + ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] - ap_id_relations = - ap_id_relations ++ + ap_id_relationships = + ap_id_relationships ++ if opts["blocking_user"] && opts["blocking_user"] == source_user do [:block] else [] end - preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships) restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts) restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 88c997b9f..9d83a9fc1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -63,11 +63,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do when action != :create ) - @relations [:follow, :unfollow] + @relationship_actions [:follow, :unfollow] @needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a - plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations) - plug(RateLimiter, [name: :relations_actions] when action in @relations) + plug( + RateLimiter, + [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions + ) + + plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions) plug(RateLimiter, [name: :app_account_creation] when action == :create) plug(:assign_account_by_id when action in @needs_account) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index 29f992a67..abfed21c8 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.Streamer.Worker do defp should_send?(%User{} = user, %Activity{} = item) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = - User.outgoing_relations_ap_ids(user, [:block, :mute, :reblog_mute]) + User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids) recipients = MapSet.new(item.recipients) -- cgit v1.2.3 From 3fa3d45dbecafb06fb7eb4f0260f610d4225e0a7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 13:05:00 +0300 Subject: [#1364] Minor improvements / comments. Further fixes of incorrect usage of "relations" as a short form of "relationships". --- lib/pleroma/activity.ex | 1 + lib/pleroma/notification.ex | 12 +++++++----- lib/pleroma/thread_mute.ex | 7 ++++--- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index bbaa561a7..5a8329e69 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -95,6 +95,7 @@ defmodule Pleroma.Activity do |> preload([activity, object: object], object: object) end + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) def user_actor(%Activity{actor: nil}), do: nil def user_actor(%Activity{} = activity) do diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 63e3e9be9..04ee510b9 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -322,6 +322,8 @@ defmodule Pleroma.Notification do @doc """ Returns a tuple with 2 elements: {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)} + + NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1 """ def get_notified_from_activity(activity, local_only \\ true) @@ -338,7 +340,7 @@ defmodule Pleroma.Notification do # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs notification_enabled_ap_ids = potential_receiver_ap_ids - |> exclude_relation_restricting_ap_ids(activity) + |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) potential_receivers = @@ -355,10 +357,10 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: {[], []} @doc "Filters out AP IDs of users basing on their relationships with activity actor user" - def exclude_relation_restricting_ap_ids([], _activity), do: [] + def exclude_relationship_restricted_ap_ids([], _activity), do: [] - def exclude_relation_restricting_ap_ids(ap_ids, %Activity{} = activity) do - relation_restricted_ap_ids = + def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do + relationship_restricted_ap_ids = activity |> Activity.user_actor() |> User.incoming_relationships_ungrouped_ap_ids([ @@ -366,7 +368,7 @@ defmodule Pleroma.Notification do :notification_mute ]) - Enum.uniq(ap_ids) -- relation_restricted_ap_ids + Enum.uniq(ap_ids) -- relationship_restricted_ap_ids end @doc "Filters out AP IDs of users who mute activity thread" diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 2b4cf02cf..a7ea13891 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -41,15 +41,16 @@ defmodule Pleroma.ThreadMute do def muter_ap_ids(context, ap_ids \\ nil) - def muter_ap_ids(context, ap_ids) when context not in [nil, ""] do + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) + def muter_ap_ids(context, _ap_ids) when is_nil(context), do: [] + + def muter_ap_ids(context, ap_ids) do context |> muters_query() |> maybe_filter_on_ap_id(ap_ids) |> Repo.all() end - def muter_ap_ids(_context, _ap_ids), do: [] - defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do where(query, [tm, u], u.ap_id in ^ap_ids) end -- cgit v1.2.3 From be5e2c4dbba63831ea6a0617556e686969b5080f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 17:01:45 +0300 Subject: Applied relationships preloading to GET /api/v1/accounts/relationships. Refactoring (User.binary_id/1). --- lib/pleroma/conversation/participation.ex | 11 ++++------- lib/pleroma/following_relationship.ex | 8 ++------ lib/pleroma/thread_mute.ex | 4 ++-- lib/pleroma/user.ex | 15 +++++++++++++++ lib/pleroma/user_relationship.ex | 9 ++------- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 +++++- 6 files changed, 30 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 693825cf5..215265fc9 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -129,21 +129,18 @@ defmodule Pleroma.Conversation.Participation do end def restrict_recipients(query, user, %{"recipients" => user_ids}) do - user_ids = + user_binary_ids = [user.id | user_ids] |> Enum.uniq() - |> Enum.reduce([], fn user_id, acc -> - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) - [user_id | acc] - end) + |> User.binary_id() conversation_subquery = __MODULE__ |> group_by([p], p.conversation_id) |> having( [p], - count(p.user_id) == ^length(user_ids) and - fragment("array_agg(?) @> ?", p.user_id, ^user_ids) + count(p.user_id) == ^length(user_binary_ids) and + fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids) ) |> select([p], %{id: p.conversation_id}) diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index dd1696136..624bddfe4 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -135,12 +135,8 @@ defmodule Pleroma.FollowingRelationship do target_users ) when is_list(source_users) and is_list(target_users) do - get_bin_ids = fn user -> - with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id - end - - source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) - target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) __MODULE__ |> where( diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index cc815430a..f657758aa 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -24,10 +24,10 @@ defmodule Pleroma.ThreadMute do end def query(user_id, context) do - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) + user_binary_id = User.binary_id(user_id) ThreadMute - |> Ecto.Query.where(user_id: ^user_id) + |> Ecto.Query.where(user_id: ^user_binary_id) |> Ecto.Query.where(context: ^context) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f74e43cce..699256a3b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -218,6 +218,21 @@ defmodule Pleroma.User do end end + @doc "Dumps id to SQL-compatible format" + def binary_id(source_id) when is_binary(source_id) do + with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do + dumped_id + else + _ -> source_id + end + end + + def binary_id(source_ids) when is_list(source_ids) do + Enum.map(source_ids, &binary_id/1) + end + + def binary_id(%User{} = user), do: binary_id(user.id) + @doc "Returns status account" @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 9423e3a42..519d2998d 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,7 +8,6 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query - alias FlakeId.Ecto.CompatType alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -84,12 +83,8 @@ defmodule Pleroma.UserRelationship do target_to_source_rel_types \\ nil ) when is_list(source_users) and is_list(target_users) do - get_bin_ids = fn user -> - with {:ok, bin_id} <- CompatType.dump(user.id), do: bin_id - end - - source_user_ids = Enum.map(source_users, &get_bin_ids.(&1)) - target_user_ids = Enum.map(target_users, &get_bin_ids.(&1)) + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 702d9e658..6b2eca1f3 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy defp find_following_rel(following_relationships, follower, following) do @@ -129,7 +130,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end def render("relationships.json", %{user: user, targets: targets}) do - render_many(targets, AccountView, "relationship.json", user: user, as: :target) + relationships_opts = StatusView.relationships_opts(user, targets) + opts = %{as: :target, user: user, relationships: relationships_opts} + + render_many(targets, AccountView, "relationship.json", opts) end defp do_render("show.json", %{user: user} = opts) do -- cgit v1.2.3 From 460e41585c2cd3f137c0f80173da60167fb318bf Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 25 Mar 2020 20:33:34 +0300 Subject: Further preloading (more endpoints), refactoring, tests. --- lib/pleroma/following_relationship.ex | 6 +++ lib/pleroma/user.ex | 5 ++- lib/pleroma/user_relationship.ex | 20 ++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 36 +++++++++++------- .../web/mastodon_api/views/notification_view.ex | 44 +++++++++++++--------- lib/pleroma/web/mastodon_api/views/status_view.ex | 29 ++++---------- 6 files changed, 86 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 624bddfe4..a9538ea4e 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -151,4 +151,10 @@ defmodule Pleroma.FollowingRelationship do ) |> Repo.all() end + + def find(following_relationships, follower, following) do + Enum.find(following_relationships, fn + fr -> fr.follower_id == follower.id and fr.following_id == following.id + end) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 699256a3b..8ccb9242d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -218,7 +218,10 @@ defmodule Pleroma.User do end end - @doc "Dumps id to SQL-compatible format" + @doc """ + Dumps Flake Id to SQL-compatible format (16-byte UUID). + E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>> + """ def binary_id(source_id) when is_binary(source_id) do with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do dumped_id diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 519d2998d..011cf6822 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias Pleroma.FollowingRelationship alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -124,6 +125,25 @@ defmodule Pleroma.UserRelationship do end end + @doc ":relationships option for StatusView / AccountView / NotificationView" + def view_relationships_option(nil = _reading_user, _actors) do + %{user_relationships: [], following_relationships: []} + end + + def view_relationships_option(%User{} = reading_user, actors) do + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) + + %{user_relationships: user_relationships, following_relationships: following_relationships} + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 6b2eca1f3..2cdfac7af 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -5,20 +5,23 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.FollowingRelationship alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView - alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy - defp find_following_rel(following_relationships, follower, following) do - Enum.find(following_relationships, fn - fr -> fr.follower_id == follower.id and fr.following_id == following.id - end) - end - def render("index.json", %{users: users} = opts) do + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + UserRelationship.view_relationships_option(opts[:for], users) + end + + opts = Map.put(opts, :relationships, relationships_opt) + users |> render_many(AccountView, "show.json", opts) |> Enum.filter(&Enum.any?/1) @@ -53,7 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do follow_state = if following_relationships do user_to_target_following_relation = - find_following_rel(following_relationships, reading_user, target) + FollowingRelationship.find(following_relationships, reading_user, target) User.get_follow_state(reading_user, target, user_to_target_following_relation) else @@ -62,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followed_by = if following_relationships do - case find_following_rel(following_relationships, target, reading_user) do + case FollowingRelationship.find(following_relationships, target, reading_user) do %{state: "accept"} -> true _ -> false end @@ -70,7 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do User.following?(target, reading_user) end - # NOTE: adjust StatusView.relationships_opts/2 if adding new relation-related flags + # NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags %{ id: to_string(target.id), following: follow_state == "accept", @@ -129,11 +132,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end - def render("relationships.json", %{user: user, targets: targets}) do - relationships_opts = StatusView.relationships_opts(user, targets) - opts = %{as: :target, user: user, relationships: relationships_opts} + def render("relationships.json", %{user: user, targets: targets} = opts) do + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + UserRelationship.view_relationships_option(user, targets) + end - render_many(targets, AccountView, "relationship.json", opts) + render_opts = %{as: :target, user: user, relationships: relationships_opt} + render_many(targets, AccountView, "relationship.json", render_opts) end defp do_render("show.json", %{user: user} = opts) do diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index e9c618496..db434271c 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -8,12 +8,13 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - def render("index.json", %{notifications: notifications, for: reading_user}) do + def render("index.json", %{notifications: notifications, for: reading_user} = opts) do activities = Enum.map(notifications, & &1.activity) parent_activities = @@ -30,21 +31,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Activity.with_preloaded_object(:left) |> Pleroma.Repo.all() - move_activities_targets = - activities - |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) - |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) - - actors = - activities - |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) - |> Enum.filter(& &1) - |> Kernel.++(move_activities_targets) + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + UserRelationship.view_relationships_option(reading_user, actors) + end opts = %{ for: reading_user, parent_activities: parent_activities, - relationships: StatusView.relationships_opts(reading_user, actors) + relationships: relationships_opt } safe_render_many(notifications, NotificationView, "show.json", opts) @@ -85,27 +93,27 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } - relationships_opts = %{relationships: opts[:relationships]} + relationships_opt = %{relationships: opts[:relationships]} case mastodon_type do "mention" -> - put_status(response, activity, reading_user, relationships_opts) + put_status(response, activity, reading_user, relationships_opt) "favourite" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opts) + put_status(response, parent_activity_fn.(), reading_user, relationships_opt) "reblog" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opts) + put_status(response, parent_activity_fn.(), reading_user, relationships_opt) "move" -> - put_target(response, activity, reading_user, relationships_opts) + put_target(response, activity, reading_user, relationships_opt) "follow" -> response "pleroma:emoji_reaction" -> response - |> put_status(parent_activity_fn.(), reading_user, relationships_opts) + |> put_status(parent_activity_fn.(), reading_user, relationships_opt) |> put_emoji(activity) _ -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0ef65b352..7b1cb7bf8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.ActivityExpiration - alias Pleroma.FollowingRelationship alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -72,24 +71,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do present?(user && user.ap_id in (object.data["announcements"] || [])) end - def relationships_opts(_reading_user = nil, _actors) do - %{user_relationships: [], following_relationships: []} - end - - def relationships_opts(reading_user, actors) do - user_relationships = - UserRelationship.dictionary( - [reading_user], - actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] - ) - - following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) - - %{user_relationships: user_relationships, following_relationships: following_relationships} - end - def render("index.json", opts) do # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) @@ -105,13 +86,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Activity.with_set_thread_muted_field(opts[:for]) |> Repo.all() - actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + relationships_opt = + if Map.has_key?(opts, :relationships) do + opts[:relationships] + else + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + UserRelationship.view_relationships_option(opts[:for], actors) + end opts = opts |> Map.put(:replied_to_activities, replied_to_activities) |> Map.put(:parent_activities, parent_activities) - |> Map.put(:relationships, relationships_opts(opts[:for], actors)) + |> Map.put(:relationships, relationships_opt) safe_render_many(activities, StatusView, "show.json", opts) end -- cgit v1.2.3 From 4cf1007a7d478a54a759d018dd7ce958a45f3977 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 15:16:54 +0100 Subject: ActivityPub: Small refactor. --- lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index dd4b04185..35c2eb133 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -129,18 +129,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # TODO rewrite in with style @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(object, meta) do - local = Keyword.fetch!(meta, :local) - {recipients, _, _} = get_recipients(object) - - {:ok, activity} = - Repo.insert(%Activity{ - data: object, - local: local, - recipients: recipients, - actor: object["actor"] - }) - - {:ok, activity, meta} + with local <- Keyword.fetch!(meta, :local), + {recipients, _, _} <- get_recipients(object), + {:ok, activity} <- + Repo.insert(%Activity{ + data: object, + local: local, + recipients: recipients, + actor: object["actor"] + }) do + {:ok, activity, meta} + end end def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do -- cgit v1.2.3 From d7aa0b645b0da48af830f252ae80458afc965281 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 14:23:19 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/object_validator.ex --- lib/pleroma/web/activity_pub/object_validator.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index cff924047..9b2889e92 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -26,8 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def stringify_keys(object) do object - |> Enum.map(fn {key, val} -> {to_string(key), val} end) - |> Enum.into(%{}) + |> Map.new(fn {key, val} -> {to_string(key), val} end) end def fetch_actor_and_object(object) do -- cgit v1.2.3 From eaacc648392e6544cd3a3b77bde266e34cebf634 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 15:33:10 +0100 Subject: Refactors. --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +-- .../web/activity_pub/object_validators/common_validations.ex | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 35c2eb133..55f4de693 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -125,8 +125,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop - @spec insert(map(), boolean(), boolean(), boolean()) :: {:ok, Activity.t()} | {:error, any()} - # TODO rewrite in with style @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(object, meta) do with local <- Keyword.fetch!(meta, :local), @@ -142,6 +140,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec insert(map(), boolean(), boolean(), boolean()) :: {:ok, Activity.t()} | {:error, any()} def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index db0e2072d..26a57f02b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -21,11 +21,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do def validate_object_presence(cng, field_name \\ :object) do cng - |> validate_change(field_name, fn field_name, actor -> - if Object.get_cached_by_ap_id(actor) do + |> validate_change(field_name, fn field_name, object -> + if Object.get_cached_by_ap_id(object) do [] else - [{field_name, "can't find user"}] + [{field_name, "can't find object"}] end end) end -- cgit v1.2.3 From 0adaab8e753b0ec22feccfc03d301073327a6d31 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 15:37:42 +0100 Subject: Bump copyright dates. --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/common_validations.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/create_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/like_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/note_validator.ex | 2 +- lib/pleroma/web/activity_pub/pipeline.ex | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 9b2889e92..dc4bce059 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidator do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 26a57f02b..b479c3918 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex index 872a12c48..908381981 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index ccbc7d071..2c1d38b06 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index eea15ce1c..fc65f1b7c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 25f29bf63..eed53cd34 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.Pipeline do -- cgit v1.2.3 From 0c60c0a76a2fcc8d13992b51704c21a35da10a0b Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 15:44:14 +0100 Subject: Validators: Use correct type for IDs. --- lib/pleroma/web/activity_pub/object_validators/create_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/like_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/note_validator.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex index 908381981..926804ce7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do @primary_key false embedded_schema do - field(:id, :string, primary_key: true) + field(:id, Types.ObjectID, primary_key: true) field(:actor, Types.ObjectID) field(:type, :string) field(:to, {:array, :string}) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 2c1d38b06..49546ceaa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do @primary_key false embedded_schema do - field(:id, :string, primary_key: true) + field(:id, Types.ObjectID, primary_key: true) field(:type, :string) field(:object, Types.ObjectID) field(:actor, Types.ObjectID) diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index fc65f1b7c..c95b622e4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do @primary_key false embedded_schema do - field(:id, :string, primary_key: true) + field(:id, Types.ObjectID, primary_key: true) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) field(:bto, {:array, :string}, default: []) -- cgit v1.2.3 From 69fc1dd69ff9d63af1785bb0701576cb5cde51f2 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 26 Mar 2020 14:45:28 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/pipeline.ex --- lib/pleroma/web/activity_pub/pipeline.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 25f29bf63..0068d60be 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -22,6 +22,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do {:ok, activity, meta} else + {:mrf_object, {:reject, _}} -> {:ok, nil, meta} e -> {:error, e} end end -- cgit v1.2.3 From 6b793d3f8336fcba5cac596f9e76d0274633f98d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 26 Mar 2020 21:54:01 +0300 Subject: Ensured no auxiliary computations (actors list preparation etc.) related to relationships preloading if no user is present (for statuses / accounts / relationships rendering). --- lib/pleroma/web/mastodon_api/views/account_view.ex | 26 +++++++++++----- .../web/mastodon_api/views/notification_view.ex | 35 ++++++++++++---------- lib/pleroma/web/mastodon_api/views/status_view.ex | 16 ++++++---- 3 files changed, 49 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2cdfac7af..0efcabc01 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -14,10 +14,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do def render("index.json", %{users: users} = opts) do relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - UserRelationship.view_relationships_option(opts[:for], users) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(opts[:for], users) end opts = Map.put(opts, :relationships, relationships_opt) @@ -134,10 +139,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do def render("relationships.json", %{user: user, targets: targets} = opts) do relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - UserRelationship.view_relationships_option(user, targets) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(user, targets) end render_opts = %{as: :target, user: user, relationships: relationships_opt} diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index db434271c..a809080fd 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -32,21 +32,26 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do |> Pleroma.Repo.all() relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - move_activities_targets = - activities - |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) - |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) - - actors = - activities - |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) - |> Enum.filter(& &1) - |> Kernel.++(move_activities_targets) - - UserRelationship.view_relationships_option(reading_user, actors) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + UserRelationship.view_relationships_option(reading_user, actors) end opts = %{ diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7b1cb7bf8..d36b9ee5c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -87,11 +87,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Repo.all() relationships_opt = - if Map.has_key?(opts, :relationships) do - opts[:relationships] - else - actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) - UserRelationship.view_relationships_option(opts[:for], actors) + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(opts[:for]) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + + UserRelationship.view_relationships_option(opts[:for], actors) end opts = -- cgit v1.2.3 From dfbc05d4965a04a82d4c4c5b8842f4117757f30e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 27 Mar 2020 08:01:03 +0300 Subject: Misc refactoring / tweaks (`ThreadMute.exists?/2`). --- lib/pleroma/thread_mute.ex | 4 ++-- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/views/notification_view.ex | 12 ++++++------ lib/pleroma/web/mastodon_api/views/status_view.ex | 7 ++++--- 4 files changed, 13 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 5768e7711..be01d541d 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -68,8 +68,8 @@ defmodule Pleroma.ThreadMute do |> Repo.delete_all() end - def check_muted(user_id, context) do + def exists?(user_id, context) do query(user_id, context) - |> Repo.all() + |> Repo.exists?() end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 091011c6b..2646b9f7b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -358,7 +358,7 @@ defmodule Pleroma.Web.CommonAPI do def thread_muted?(%{id: nil} = _user, _activity), do: false def thread_muted?(user, activity) do - ThreadMute.check_muted(user.id, activity.data["context"]) != [] + ThreadMute.exists?(user.id, activity.data["context"]) end def report(user, %{"account_id" => account_id} = data) do diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index a809080fd..89f5734ff 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -98,27 +98,27 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } - relationships_opt = %{relationships: opts[:relationships]} + render_opts = %{relationships: opts[:relationships]} case mastodon_type do "mention" -> - put_status(response, activity, reading_user, relationships_opt) + put_status(response, activity, reading_user, render_opts) "favourite" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opt) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "reblog" -> - put_status(response, parent_activity_fn.(), reading_user, relationships_opt) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "move" -> - put_target(response, activity, reading_user, relationships_opt) + put_target(response, activity, reading_user, render_opts) "follow" -> response "pleroma:emoji_reaction" -> response - |> put_status(parent_activity_fn.(), reading_user, relationships_opt) + |> put_status(parent_activity_fn.(), reading_user, render_opts) |> put_emoji(activity) _ -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d36b9ee5c..440eef4ba 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -228,9 +228,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end thread_muted? = - case activity.thread_muted? do - thread_muted? when is_boolean(thread_muted?) -> thread_muted? - nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false + cond do + is_nil(opts[:for]) -> false + is_boolean(activity.thread_muted?) -> activity.thread_muted? + true -> CommonAPI.thread_muted?(opts[:for], activity) end attachment_data = object.data["attachment"] || [] -- cgit v1.2.3 From eb9744cadea7191b088ddaadfbd5fa4d4fd45090 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 14:42:30 +0300 Subject: activities generation tasks --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 33f1705df..51850abb5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Application do defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) - defp streamer_child(:test), do: [] + defp streamer_child(env) when env in [:test, :benchmark], do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] -- cgit v1.2.3 From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: sync with develop --- lib/mix/pleroma.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..4dfcc32e7 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -5,6 +5,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do + Mix.Task.run("app.start") Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do -- cgit v1.2.3 From 1fcdcb12a717fa3dbd54a5c3778bd216df6449ad Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 12:47:12 +0300 Subject: updating gun with bug fix https://github.com/ninenines/gun/issues/222 --- lib/pleroma/pool/connections.ex | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 91102faf7..4d4ba913c 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -167,29 +167,20 @@ defmodule Pleroma.Pool.Connections do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end - defp find_conn_from_gun_info(conns, pid) do - # TODO: temp fix for gun MatchError https://github.com/ninenines/gun/issues/222 - # TODO: REMOVE LATER - try do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - find_conn(conns, pid, key) - rescue - MatcheError -> find_conn(conns, pid) - end - end - @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + state = - with {key, conn} <- find_conn_from_gun_info(state.conns, conn_pid), + with {key, conn} <- find_conn(state.conns, conn_pid, key), {true, key} <- {Process.alive?(conn_pid), key} do put_in(state.conns[key], %{ conn -- cgit v1.2.3 From b607ae1a1c0ef6557094ec0fb10ba2d19d621f7f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 13:50:00 +0300 Subject: removing grouped reports admin api endpoint --- lib/pleroma/web/activity_pub/utils.ex | 96 ----------------------- lib/pleroma/web/admin_api/admin_api_controller.ex | 8 -- lib/pleroma/web/admin_api/views/report_view.ex | 28 +------ lib/pleroma/web/router.ex | 1 - 4 files changed, 1 insertion(+), 132 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c65bbed67..2d685ecc0 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -795,102 +795,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do ActivityPub.fetch_activities([], params, :offset) end - def parse_report_group(activity) do - reports = get_reports_by_status_id(activity["id"]) - max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) - actors = Enum.map(reports, & &1.user_actor) - [%{data: %{"object" => [account_id | _]}} | _] = reports - - account = - AccountView.render("show.json", %{ - user: User.get_by_ap_id(account_id) - }) - - status = get_status_data(activity) - - %{ - date: max_date.data["published"], - account: account, - status: status, - actors: Enum.uniq(actors), - reports: reports - } - end - - defp get_status_data(status) do - case status["deleted"] do - true -> - %{ - "id" => status["id"], - "deleted" => true - } - - _ -> - Activity.get_by_ap_id(status["id"]) - end - end - - def get_reports_by_status_id(ap_id) do - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - where: fragment("(?)->'object' @> ?", a.data, ^[%{id: ap_id}]), - or_where: fragment("(?)->'object' @> ?", a.data, ^[ap_id]) - ) - |> Activity.with_preloaded_user_actor() - |> Repo.all() - end - - @spec get_reports_grouped_by_status([String.t()]) :: %{ - required(:groups) => [ - %{ - required(:date) => String.t(), - required(:account) => %{}, - required(:status) => %{}, - required(:actors) => [%User{}], - required(:reports) => [%Activity{}] - } - ] - } - def get_reports_grouped_by_status(activity_ids) do - parsed_groups = - activity_ids - |> Enum.map(fn id -> - id - |> build_flag_object() - |> parse_report_group() - end) - - %{ - groups: parsed_groups - } - end - - @spec get_reported_activities() :: [ - %{ - required(:activity) => String.t(), - required(:date) => String.t() - } - ] - def get_reported_activities do - reported_activities_query = - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - select: %{ - activity: fragment("jsonb_array_elements((? #- '{object,0}')->'object')", a.data) - }, - group_by: fragment("activity") - ) - - from(a in subquery(reported_activities_query), - distinct: true, - select: %{ - id: fragment("COALESCE(?->>'id'::text, ? #>> '{}')", a.activity, a.activity) - } - ) - |> Repo.all() - |> Enum.map(& &1.id) - end - def update_report_state(%Activity{} = activity, state) when state in @strip_status_report_states do {:ok, stripped_activity} = strip_report_status_data(activity) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0368df1e9..ca5439920 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -715,14 +715,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{reports: reports}) end - def list_grouped_reports(conn, _params) do - statuses = Utils.get_reported_activities() - - conn - |> put_view(ReportView) - |> render("index_grouped.json", Utils.get_reports_grouped_by_status(statuses)) - end - def report_show(conn, %{"id" => id}) do with %Activity{} = report <- Activity.get_by_id(id) do conn diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index fc8733ce8..ca0bcebc7 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view - alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.AdminAPI.Report @@ -44,32 +44,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do } end - def render("index_grouped.json", %{groups: groups}) do - reports = - Enum.map(groups, fn group -> - status = - case group.status do - %Activity{} = activity -> StatusView.render("show.json", %{activity: activity}) - _ -> group.status - end - - %{ - date: group[:date], - account: group[:account], - status: Map.put_new(status, "deleted", false), - actors: Enum.map(group[:actors], &merge_account_views/1), - reports: - group[:reports] - |> Enum.map(&Report.extract_report_info(&1)) - |> Enum.map(&render(__MODULE__, "show.json", &1)) - } - end) - - %{ - reports: reports - } - end - def render("index_notes.json", %{notes: notes}) when is_list(notes) do Enum.map(notes, &render(__MODULE__, "show_note.json", &1)) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a22f744c1..5a0902739 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -186,7 +186,6 @@ defmodule Pleroma.Web.Router do patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email) get("/reports", AdminAPIController, :list_reports) - get("/grouped_reports", AdminAPIController, :list_grouped_reports) get("/reports/:id", AdminAPIController, :report_show) patch("/reports", AdminAPIController, :reports_update) post("/reports/:id/notes", AdminAPIController, :report_notes_create) -- cgit v1.2.3 From f6835333be745cd411b5d2571c304fc7a16d645e Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 12:55:25 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/transmogrifier.ex --- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index dbb14e9aa..23148b2a0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -615,8 +615,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with {_, {:ok, cast_data_sym}} <- {:casting_data, data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)}, - {_, cast_data} <- - {:stringify_keys, ObjectValidator.stringify_keys(cast_data_sym |> Map.from_struct())}, + cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)), :ok <- ObjectValidator.fetch_actor_and_object(cast_data), {_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)}, {_, {:ok, cast_data}} <- -- cgit v1.2.3 From d191b0942f64a32a2bf450318fac85981aa17c83 Mon Sep 17 00:00:00 2001 From: kPherox Date: Tue, 31 Mar 2020 22:48:42 +0900 Subject: Remove no longer used function --- lib/pleroma/user.ex | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9aa54057..6644d6b66 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1983,17 +1983,6 @@ defmodule Pleroma.User do def fields(%{fields: fields}), do: fields - def sanitized_fields(%User{} = user) do - user - |> User.fields() - |> Enum.map(fn %{"name" => name, "value" => value} -> - %{ - "name" => name, - "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) - } - end) - end - def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Pleroma.Config.get([:instance, limit_name], 0) -- cgit v1.2.3 From 643f15e77b7cdaaf2c22a876c98e5680edc32dc3 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 16:11:38 +0200 Subject: Validators: ObjectID is an http uri. --- .../web/activity_pub/object_validators/types/object.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object.ex b/lib/pleroma/web/activity_pub/object_validators/types/object.ex index 92fc13ba8..8e70effe4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object.ex @@ -4,12 +4,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do def type, do: :string def cast(object) when is_binary(object) do - {:ok, object} + with %URI{ + scheme: scheme, + host: host + } + when scheme in ["https", "http"] and not is_nil(host) <- + URI.parse(object) do + {:ok, object} + else + _ -> + :error + end end - def cast(%{"id" => object}) when is_binary(object) do - {:ok, object} - end + def cast(%{"id" => object}), do: cast(object) def cast(_) do :error -- cgit v1.2.3 From 057438a657eaadb963e006b84b890ae4f8441808 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 17:56:05 +0200 Subject: CommonAPI: DRY up a bit. --- lib/pleroma/web/common_api/common_api.ex | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index f882f9fcb..74adcca55 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -112,8 +112,22 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec favorite(User.t(), binary()) :: {:ok, Activity.t()} | {:error, any()} + @spec favorite(User.t(), binary()) :: {:ok, Activity.t() | :already_liked} | {:error, any()} def favorite(%User{} = user, id) do + case favorite_helper(user, id) do + {:ok, _} = res -> + res + + {:error, :not_found} = res -> + res + + {:error, e} -> + Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}") + {:error, dgettext("errors", "Could not favorite")} + end + end + + def favorite_helper(user, id) do with {_, %Activity{object: object}} <- {:find_object, Activity.get_by_id_with_object(id)}, {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, {_, {:ok, %Activity{} = activity, _meta}} <- @@ -138,13 +152,11 @@ defmodule Pleroma.Web.CommonAPI do if {:object, {"already liked by this actor", []}} in changeset.errors do {:ok, :already_liked} else - Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}") - {:error, dgettext("errors", "Could not favorite"), e} + {:error, e} end e -> - Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}") - {:error, dgettext("errors", "Could not favorite"), e} + {:error, e} end end -- cgit v1.2.3 From 0be1fa0a8695df87a8b22279b885956943e33796 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 17:00:48 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/transmogrifier.ex --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 23148b2a0..fb41ec8e9 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1291,6 +1291,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end defp maybe_add_recipients_from_object(_) do - {:error, "No referenced object"} + {:error, :no_object} end end -- cgit v1.2.3 From 288f2b5a7c728959d43205a97d5225b34b5b8161 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 17:00:55 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/transmogrifier.ex --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index fb41ec8e9..a3529f09b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1267,7 +1267,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end defp maybe_add_context_from_object(_) do - {:error, "No referenced object"} + {:error, :no_context} end defp maybe_add_recipients_from_object(%{"object" => object} = data) do -- cgit v1.2.3 From ecac57732a063c1ad01aeb5aa4eb9853b6f904e9 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 19:16:45 +0200 Subject: Transmogrifier: Only add context if it really is onne. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a3529f09b..f82142979 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1255,14 +1255,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do do: {:ok, data} defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(object) do - if object = Object.normalize(object) do - data = - data - |> Map.put("context", object.data["context"]) - - {:ok, data} + with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do + {:ok, Map.put(data, "context", context)} else - {:error, "No context on referenced object"} + _ -> + {:error, :no_context} end end -- cgit v1.2.3 From 1b323ce1c668c6a26617a05dcc12ee255c764e88 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 17:28:18 +0000 Subject: Apply suggestion to lib/pleroma/web/activity_pub/transmogrifier.ex --- lib/pleroma/web/activity_pub/transmogrifier.ex | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f82142979..a18ece6e7 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1267,24 +1267,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:error, :no_context} end - defp maybe_add_recipients_from_object(%{"object" => object} = data) do - to = data["to"] || [] - cc = data["cc"] || [] + defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data), do: {:ok, data} - if to == [] && cc == [] do - if object = Object.normalize(object) do + defp maybe_add_recipients_from_object(%{"object" => object} = data) do + case Object.normalize(object) do + %{data: {"actor" => actor}} -> data = data - |> Map.put("to", [object.data["actor"]]) - |> Map.put("cc", cc) + |> Map.put("to", [actor]) + |> Map.put("cc", data["cc"] || []) {:ok, data} - else - {:error, "No actor on referenced object"} - end - else - {:ok, data} - end + nil -> {:error, :no_object} + _ -> {:error, :no_actor} end defp maybe_add_recipients_from_object(_) do -- cgit v1.2.3 From c982093cc2f538e8ef9dde365e163a944c6cb6d0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 31 Mar 2020 19:33:41 +0200 Subject: Transmogrifier: Fix BAD code by RINPATCH --- lib/pleroma/web/activity_pub/transmogrifier.ex | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a18ece6e7..a4b385cd5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1267,19 +1267,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:error, :no_context} end - defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data), do: {:ok, data} + defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data), + do: {:ok, data} defp maybe_add_recipients_from_object(%{"object" => object} = data) do case Object.normalize(object) do - %{data: {"actor" => actor}} -> + %{data: %{"actor" => actor}} -> data = data |> Map.put("to", [actor]) |> Map.put("cc", data["cc"] || []) {:ok, data} - nil -> {:error, :no_object} - _ -> {:error, :no_actor} + + nil -> + {:error, :no_object} + + _ -> + {:error, :no_actor} + end end defp maybe_add_recipients_from_object(_) do -- cgit v1.2.3 From 7408f003a663c5f634cabad963c0446ba54810bf Mon Sep 17 00:00:00 2001 From: kPherox Date: Tue, 31 Mar 2020 11:13:53 +0000 Subject: Use `Pleroma.Formatter.linkify` instead of `AutoLinker.link` --- lib/pleroma/user.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 6644d6b66..c29935871 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -16,6 +16,7 @@ defmodule Pleroma.User do alias Pleroma.Conversation.Participation alias Pleroma.Delivery alias Pleroma.FollowingRelationship + alias Pleroma.Formatter alias Pleroma.HTML alias Pleroma.Keys alias Pleroma.Notification @@ -456,7 +457,7 @@ defmodule Pleroma.User do fields = raw_fields - |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + |> Enum.map(fn f -> Map.update!(f, "value", &parse_fields(&1)) end) changeset |> put_change(:raw_fields, raw_fields) @@ -466,6 +467,12 @@ defmodule Pleroma.User do end end + defp parse_fields(value) do + value + |> Formatter.linkify(mentions_format: :full) + |> elem(0) + end + defp put_change_if_present(changeset, map_field, value_function) do if value = get_change(changeset, map_field) do with {:ok, new_value} <- value_function.(value) do -- cgit v1.2.3 From b30fb1f3bbf8fb8e49cc5276225dc09771c79477 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 29 Mar 2020 22:30:50 +0200 Subject: User: Fix use of source_data in profile_url/1 --- lib/pleroma/user.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9aa54057..ca0bfca11 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -305,7 +305,8 @@ defmodule Pleroma.User do end end - def profile_url(%User{source_data: %{"url" => url}}), do: url + def profile_url(%User{uri: url}) when url != nil, do: url + def profile_url(%User{source_data: %{"url" => url}}) when is_binary(url), do: url def profile_url(%User{ap_id: ap_id}), do: ap_id def profile_url(_), do: nil @@ -314,7 +315,7 @@ defmodule Pleroma.User do def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" - @spec ap_following(User.t()) :: Sring.t() + @spec ap_following(User.t()) :: String.t() def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" -- cgit v1.2.3 From 185520d1b4d3fdf8ecde7814faec92bbb531ce59 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 30 Mar 2020 02:01:09 +0200 Subject: Provide known-good user.uri, remove User.profile_url/1 --- lib/pleroma/user.ex | 5 ----- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 4 ++-- lib/pleroma/web/metadata/opengraph.ex | 2 +- .../web/templates/static_fe/static_fe/_user_card.html.eex | 2 +- .../web/templates/static_fe/static_fe/profile.html.eex | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ca0bfca11..ff828aa17 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -305,11 +305,6 @@ defmodule Pleroma.User do end end - def profile_url(%User{uri: url}) when url != nil, do: url - def profile_url(%User{source_data: %{"url" => url}}) when is_binary(url), do: url - def profile_url(%User{ap_id: ap_id}), do: ap_id - def profile_url(_), do: nil - def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9c0f5d771..53b6ad654 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1379,6 +1379,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec get_actor_url(any()) :: binary() | nil + defp get_actor_url(url) when is_binary(url), do: url + defp get_actor_url(%{"href" => href}) when is_binary(href), do: href + + defp get_actor_url(url) when is_list(url) do + url + |> List.first() + |> get_actor_url() + end + + defp get_actor_url(_url), do: nil + defp object_to_user_data(data) do avatar = data["icon"]["url"] && @@ -1408,6 +1420,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do user_data = %{ ap_id: data["id"], + uri: get_actor_url(data["url"]), ap_enabled: true, source_data: data, banner: banner, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 0efcabc01..c482bba64 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: User.profile_url(user) + url: user.uri || user.ap_id } end @@ -207,7 +207,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do following_count: following_count, statuses_count: user.note_count, note: user.bio || "", - url: User.profile_url(user), + url: user.uri || user.ap_id, avatar: image, avatar_static: image, header: header, diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 21446ac77..68c871e71 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do property: "og:title", content: Utils.user_name_string(user) ], []}, - {:meta, [property: "og:url", content: User.profile_url(user)], []}, + {:meta, [property: "og:url", content: user.uri || user.ap_id], []}, {:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:type", content: "website"], []}, {:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []}, diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex index c7789f9ac..2a7582d45 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex @@ -1,5 +1,5 @@
- +
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 94063c92d..e7d2aecad 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -8,7 +8,7 @@ <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> | - <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %> + <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>

<%= raw @user.bio %>

-- cgit v1.2.3 From 94ddbe4098e167f9537d168261a6cc76fa17508b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 1 Apr 2020 09:55:05 +0300 Subject: restrict remote users from indexing --- lib/pleroma/web/metadata.ex | 7 ++++++- lib/pleroma/web/metadata/restrict_indexing.ex | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/metadata/restrict_indexing.ex (limited to 'lib') diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index c9aac27dc..a9f70c43e 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -6,7 +6,12 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML def build_tags(params) do - Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc -> + providers = [ + Pleroma.Web.Metadata.Providers.RestrictIndexing + | Pleroma.Config.get([__MODULE__, :providers], []) + ] + + Enum.reduce(providers, "", fn parser, acc -> rendered_html = params |> parser.build_tags() diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex new file mode 100644 index 000000000..f15607896 --- /dev/null +++ b/lib/pleroma/web/metadata/restrict_indexing.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do + @behaviour Pleroma.Web.Metadata.Providers.Provider + + @moduledoc """ + Restricts indexing of remote users. + """ + + @impl true + def build_tags(%{user: %{local: false}}) do + [ + {:meta, + [ + name: "robots", + content: "noindex, noarchive" + ], []} + ] + end + + @impl true + def build_tags(%{user: %{local: true}}), do: [] +end -- cgit v1.2.3 From 037b49c415060b4c7ad5a570da80857b4d2c43f1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 1 Apr 2020 16:10:17 +0200 Subject: Validators: Correct ObjectID filename --- .../activity_pub/object_validators/types/object.ex | 33 ---------------------- .../object_validators/types/object_id.ex | 33 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 33 deletions(-) delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/object.ex create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/object_id.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object.ex b/lib/pleroma/web/activity_pub/object_validators/types/object.ex deleted file mode 100644 index 8e70effe4..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/object.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do - use Ecto.Type - - def type, do: :string - - def cast(object) when is_binary(object) do - with %URI{ - scheme: scheme, - host: host - } - when scheme in ["https", "http"] and not is_nil(host) <- - URI.parse(object) do - {:ok, object} - else - _ -> - :error - end - end - - def cast(%{"id" => object}), do: cast(object) - - def cast(_) do - :error - end - - def dump(data) do - {:ok, data} - end - - def load(data) do - {:ok, data} - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex new file mode 100644 index 000000000..8e70effe4 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do + use Ecto.Type + + def type, do: :string + + def cast(object) when is_binary(object) do + with %URI{ + scheme: scheme, + host: host + } + when scheme in ["https", "http"] and not is_nil(host) <- + URI.parse(object) do + {:ok, object} + else + _ -> + :error + end + end + + def cast(%{"id" => object}), do: cast(object) + + def cast(_) do + :error + end + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end -- cgit v1.2.3 From 2d64500a9dee8bc53c988719bde1c1f4f41575b7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 1 Apr 2020 20:26:33 +0300 Subject: error improvement for email_invite endpoint --- lib/pleroma/web/admin_api/admin_api_controller.ex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index ca5439920..7b442f6e1 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -576,9 +576,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do @doc "Sends registration invite via email" def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) do - with true <- - Config.get([:instance, :invites_enabled]) && - !Config.get([:instance, :registrations_open]), + with {_, false} <- {:registrations_open, Config.get([:instance, :registrations_open])}, + {_, true} <- {:invites_enabled, Config.get([:instance, :invites_enabled])}, {:ok, invite_token} <- UserInviteToken.create_invite(), email <- Pleroma.Emails.UserEmail.user_invitation_email( @@ -589,6 +588,18 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do ), {:ok, _} <- Pleroma.Emails.Mailer.deliver(email) do json_response(conn, :no_content, "") + else + {:registrations_open, _} -> + errors( + conn, + {:error, "To send invites you need set `registrations_open` option to false."} + ) + + {:invites_enabled, _} -> + errors( + conn, + {:error, "To send invites you need set `invites_enabled` option to true."} + ) end end -- cgit v1.2.3 From 23219e6fb3163bfac07fb5fb1b2602dcd27e47c2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 1 Apr 2020 23:00:59 +0400 Subject: Add OpenAPI --- lib/pleroma/web/api_spec.ex | 30 +++++++ .../web/api_spec/operations/app_operation.ex | 94 ++++++++++++++++++++++ .../web/api_spec/schemas/app_create_request.ex | 33 ++++++++ .../web/api_spec/schemas/app_create_response.ex | 33 ++++++++ .../web/mastodon_api/controllers/app_controller.ex | 9 ++- lib/pleroma/web/oauth/scopes.ex | 7 +- lib/pleroma/web/router.ex | 11 +++ 7 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/web/api_spec.ex create mode 100644 lib/pleroma/web/api_spec/operations/app_operation.ex create mode 100644 lib/pleroma/web/api_spec/schemas/app_create_request.ex create mode 100644 lib/pleroma/web/api_spec/schemas/app_create_response.ex (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex new file mode 100644 index 000000000..22f76d4bf --- /dev/null +++ b/lib/pleroma/web/api_spec.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec do + alias OpenApiSpex.OpenApi + alias Pleroma.Web.Endpoint + alias Pleroma.Web.Router + + @behaviour OpenApi + + @impl OpenApi + def spec do + %OpenApi{ + servers: [ + # Populate the Server info from a phoenix endpoint + OpenApiSpex.Server.from_endpoint(Endpoint) + ], + info: %OpenApiSpex.Info{ + title: "Pleroma", + description: Application.spec(:pleroma, :description) |> to_string(), + version: Application.spec(:pleroma, :vsn) |> to_string() + }, + # populate the paths from a phoenix router + paths: OpenApiSpex.Paths.from_router(Router) + } + # discover request/response schemas from path specs + |> OpenApiSpex.resolve_schema_modules() + end +end diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex new file mode 100644 index 000000000..2a4958acf --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -0,0 +1,94 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.AppOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.AppCreateRequest + alias Pleroma.Web.ApiSpec.Schemas.AppCreateResponse + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec create_operation() :: Operation.t() + def create_operation do + %Operation{ + tags: ["apps"], + summary: "Create an application", + description: "Create a new application to obtain OAuth2 credentials", + operationId: "AppController.create", + requestBody: + Operation.request_body("Parameters", "application/json", AppCreateRequest, required: true), + responses: %{ + 200 => Operation.response("App", "application/json", AppCreateResponse), + 422 => + Operation.response( + "Unprocessable Entity", + "application/json", + %Schema{ + type: :object, + description: + "If a required parameter is missing or improperly formatted, the request will fail.", + properties: %{ + error: %Schema{type: :string} + }, + example: %{ + "error" => "Validation failed: Redirect URI must be an absolute URI." + } + } + ) + } + } + end + + def verify_credentials_operation do + %Operation{ + tags: ["apps"], + summary: "Verify your app works", + description: "Confirm that the app's OAuth2 credentials work.", + operationId: "AppController.verify_credentials", + parameters: [ + Operation.parameter(:authorization, :header, :string, "Bearer ", required: true) + ], + responses: %{ + 200 => + Operation.response("App", "application/json", %Schema{ + type: :object, + description: + "If the Authorization header was provided with a valid token, you should see your app returned as an Application entity.", + properties: %{ + name: %Schema{type: :string}, + vapid_key: %Schema{type: :string}, + website: %Schema{type: :string, nullable: true} + }, + example: %{ + "name" => "My App", + "vapid_key" => + "BCk-QqERU0q-CfYZjcuB6lnyyOYfJ2AifKqfeGIm7Z-HiTU5T9eTG5GxVA0_OH5mMlI4UkkDTpaZwozy0TzdZ2M=", + "website" => "https://myapp.com/" + } + }), + 422 => + Operation.response( + "Unauthorized", + "application/json", + %Schema{ + type: :object, + description: + "If the Authorization header contains an invalid token, is malformed, or is not present, an error will be returned indicating an authorization failure.", + properties: %{ + error: %Schema{type: :string} + }, + example: %{ + "error" => "The access token is invalid." + } + } + ) + } + } + end +end diff --git a/lib/pleroma/web/api_spec/schemas/app_create_request.ex b/lib/pleroma/web/api_spec/schemas/app_create_request.ex new file mode 100644 index 000000000..8a83abef3 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/app_create_request.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.AppCreateRequest do + alias OpenApiSpex.Schema + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "AppCreateRequest", + description: "POST body for creating an app", + type: :object, + properties: %{ + client_name: %Schema{type: :string, description: "A name for your application."}, + redirect_uris: %Schema{ + type: :string, + description: + "Where the user should be redirected after authorization. To display the authorization code to the user instead of redirecting to a web page, use `urn:ietf:wg:oauth:2.0:oob` in this parameter." + }, + scopes: %Schema{ + type: :string, + description: "Space separated list of scopes. If none is provided, defaults to `read`." + }, + website: %Schema{type: :string, description: "A URL to the homepage of your app"} + }, + required: [:client_name, :redirect_uris], + example: %{ + "client_name" => "My App", + "redirect_uris" => "https://myapp.com/auth/callback", + "website" => "https://myapp.com/" + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/app_create_response.ex b/lib/pleroma/web/api_spec/schemas/app_create_response.ex new file mode 100644 index 000000000..f290fb031 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/app_create_response.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.AppCreateResponse do + alias OpenApiSpex.Schema + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "AppCreateResponse", + description: "Response schema for an app", + type: :object, + properties: %{ + id: %Schema{type: :string}, + name: %Schema{type: :string}, + client_id: %Schema{type: :string}, + client_secret: %Schema{type: :string}, + redirect_uri: %Schema{type: :string}, + vapid_key: %Schema{type: :string}, + website: %Schema{type: :string, nullable: true} + }, + example: %{ + "id" => "123", + "name" => "My App", + "client_id" => "TWhM-tNSuncnqN7DBJmoyeLnk6K3iJJ71KKXxgL1hPM", + "client_secret" => "ZEaFUFmF0umgBX1qKJDjaU99Q31lDkOU8NutzTOoliw", + "vapid_key" => + "BCk-QqERU0q-CfYZjcuB6lnyyOYfJ2AifKqfeGIm7Z-HiTU5T9eTG5GxVA0_OH5mMlI4UkkDTpaZwozy0TzdZ2M=", + "website" => "https://myapp.com/" + } + }) +end diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index 5e2871f18..005c60444 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -14,17 +14,20 @@ defmodule Pleroma.Web.MastodonAPI.AppController do action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read"]} when action == :verify_credentials) + plug(OpenApiSpex.Plug.CastAndValidate) @local_mastodon_name "Mastodon-Local" + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.AppOperation + @doc "POST /api/v1/apps" - def create(conn, params) do + def create(%{body_params: params} = conn, _params) do scopes = Scopes.fetch_scopes(params, ["read"]) app_attrs = params - |> Map.drop(["scope", "scopes"]) - |> Map.put("scopes", scopes) + |> Map.take([:client_name, :redirect_uris, :website]) + |> Map.put(:scopes, scopes) with cs <- App.register_changeset(%App{}, app_attrs), false <- cs.changes[:client_name] == @local_mastodon_name, diff --git a/lib/pleroma/web/oauth/scopes.ex b/lib/pleroma/web/oauth/scopes.ex index 8ecf901f3..1023f16d4 100644 --- a/lib/pleroma/web/oauth/scopes.ex +++ b/lib/pleroma/web/oauth/scopes.ex @@ -15,7 +15,12 @@ defmodule Pleroma.Web.OAuth.Scopes do Note: `scopes` is used by Mastodon — supporting it but sticking to OAuth's standard `scope` wherever we control it """ - @spec fetch_scopes(map(), list()) :: list() + @spec fetch_scopes(map() | struct(), list()) :: list() + + def fetch_scopes(%Pleroma.Web.ApiSpec.Schemas.AppCreateRequest{scopes: scopes}, default) do + parse_scopes(scopes, default) + end + def fetch_scopes(params, default) do parse_scopes(params["scope"] || params["scopes"], default) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5a0902739..3ecd59cd1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureUserKeyPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :authenticated_api do @@ -44,6 +45,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :admin_api do @@ -61,6 +63,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Plugs.UserIsAdminPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :mastodon_html do @@ -94,10 +97,12 @@ defmodule Pleroma.Web.Router do pipeline :config do plug(:accepts, ["json", "xml"]) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :pleroma_api do plug(:accepts, ["html", "json"]) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :mailbox_preview do @@ -500,6 +505,12 @@ defmodule Pleroma.Web.Router do ) end + scope "/api" do + pipe_through(:api) + + get("/openapi", OpenApiSpex.Plug.RenderSpec, []) + end + scope "/api", Pleroma.Web, as: :authenticated_twitter_api do pipe_through(:authenticated_api) -- cgit v1.2.3 From 0aa24a150bbb153f55ca92dfb595385b4fe3839c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 2 Apr 2020 17:33:23 +0400 Subject: Add oAuth --- lib/pleroma/web/api_spec.ex | 16 +++++++++++++++- lib/pleroma/web/api_spec/operations/app_operation.ex | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 22f76d4bf..41e48a085 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -22,7 +22,21 @@ defmodule Pleroma.Web.ApiSpec do version: Application.spec(:pleroma, :vsn) |> to_string() }, # populate the paths from a phoenix router - paths: OpenApiSpex.Paths.from_router(Router) + paths: OpenApiSpex.Paths.from_router(Router), + components: %OpenApiSpex.Components{ + securitySchemes: %{ + "oAuth" => %OpenApiSpex.SecurityScheme{ + type: "oauth2", + flows: %OpenApiSpex.OAuthFlows{ + password: %OpenApiSpex.OAuthFlow{ + authorizationUrl: "/oauth/authorize", + tokenUrl: "/oauth/token", + scopes: %{"read" => "read"} + } + } + } + } + } } # discover request/response schemas from path specs |> OpenApiSpex.resolve_schema_modules() diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex index 2a4958acf..41d56693a 100644 --- a/lib/pleroma/web/api_spec/operations/app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -51,8 +51,10 @@ defmodule Pleroma.Web.ApiSpec.AppOperation do summary: "Verify your app works", description: "Confirm that the app's OAuth2 credentials work.", operationId: "AppController.verify_credentials", - parameters: [ - Operation.parameter(:authorization, :header, :string, "Bearer ", required: true) + security: [ + %{ + "oAuth" => ["read"] + } ], responses: %{ 200 => -- cgit v1.2.3 From aa78325117c879ecb7ec76383c239078275adbd9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 2 Apr 2020 19:23:30 +0300 Subject: [#2323] Fixed a typo causing /accounts/relationships to render default relationships. Improved the tests. --- lib/pleroma/web/mastodon_api/views/account_view.ex | 8 +++++--- lib/pleroma/web/mastodon_api/views/notification_view.ex | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index c482bba64..99e62f580 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -13,16 +13,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.Web.MediaProxy def render("index.json", %{users: users} = opts) do + reading_user = opts[:for] + relationships_opt = cond do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> - UserRelationship.view_relationships_option(opts[:for], users) + UserRelationship.view_relationships_option(reading_user, users) end opts = Map.put(opts, :relationships, relationships_opt) @@ -143,7 +145,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(user) -> UserRelationship.view_relationships_option(nil, []) true -> diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 89f5734ff..ae87d4701 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 82326986c..cea76e735 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -72,6 +72,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render("index.json", opts) do + reading_user = opts[:for] + # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) replied_to_activities = get_replied_to_activities(activities) @@ -82,8 +84,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.map(&Object.normalize(&1).data["id"]) |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object(:left) - |> Activity.with_preloaded_bookmark(opts[:for]) - |> Activity.with_set_thread_muted_field(opts[:for]) + |> Activity.with_preloaded_bookmark(reading_user) + |> Activity.with_set_thread_muted_field(reading_user) |> Repo.all() relationships_opt = @@ -91,13 +93,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do Map.has_key?(opts, :relationships) -> opts[:relationships] - is_nil(opts[:for]) -> + is_nil(reading_user) -> UserRelationship.view_relationships_option(nil, []) true -> actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) - UserRelationship.view_relationships_option(opts[:for], actors) + UserRelationship.view_relationships_option(reading_user, actors) end opts = -- cgit v1.2.3 From b59ac37b2c09d5dc80b59bd3a2aea36989bee713 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 Apr 2020 10:45:25 +0300 Subject: tests for emoji mix task --- lib/mix/tasks/pleroma/emoji.ex | 80 ++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 429d763c7..cdffa88b2 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -14,8 +14,8 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, [], []} = parse_global_opts(args) - manifest = - fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest()) + url_or_path = options[:manifest] || default_manifest() + manifest = fetch_manifest(url_or_path) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -40,9 +40,9 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, pack_names, []} = parse_global_opts(args) - manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest() + url_or_path = options[:manifest] || default_manifest() - manifest = fetch_manifest(manifest_url) + manifest = fetch_manifest(url_or_path) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -75,7 +75,10 @@ defmodule Mix.Tasks.Pleroma.Emoji do end # The url specified in files should be in the same directory - files_url = Path.join(Path.dirname(manifest_url), pack["files"]) + files_url = + url_or_path + |> Path.dirname() + |> Path.join(pack["files"]) IO.puts( IO.ANSI.format([ @@ -133,38 +136,51 @@ defmodule Mix.Tasks.Pleroma.Emoji do end end - def run(["gen-pack", src]) do + def run(["gen-pack" | args]) do start_pleroma() - proposed_name = Path.basename(src) |> Path.rootname() - name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) - # If there's no name, use the default one - name = if String.length(name) > 0, do: name, else: proposed_name + {opts, [src], []} = + OptionParser.parse( + args, + strict: [ + name: :string, + license: :string, + homepage: :string, + description: :string, + files: :string, + extensions: :string + ] + ) - license = String.trim(IO.gets("License: ")) - homepage = String.trim(IO.gets("Homepage: ")) - description = String.trim(IO.gets("Description: ")) + proposed_name = Path.basename(src) |> Path.rootname() + name = get_option(opts, :name, "Pack name:", proposed_name) + license = get_option(opts, :license, "License:") + homepage = get_option(opts, :homepage, "Homepage:") + description = get_option(opts, :description, "Description:") - proposed_files_name = "#{name}.json" - files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: ")) - files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name + proposed_files_name = "#{name}_files.json" + files_name = get_option(opts, :files, "Save file list to:", proposed_files_name) default_exts = [".png", ".gif"] - default_exts_str = Enum.join(default_exts, " ") - exts = - String.trim( - IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ") + custom_exts = + get_option( + opts, + :extensions, + "Emoji file extensions (separated with spaces):", + Enum.join(default_exts, " ") ) + |> String.split(" ", trim: true) exts = - if String.length(exts) > 0 do - String.split(exts, " ") - |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end) - else + if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do default_exts + else + custom_exts end + IO.puts("Using #{Enum.join(exts, " ")} extensions") + IO.puts("Downloading the pack and generating SHA256") binary_archive = Tesla.get!(client(), src).body @@ -194,14 +210,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(""" #{files_name} has been created and contains the list of all found emojis in the pack. - Please review the files in the remove those not needed. + Please review the files in the pack and remove those not needed. """) - if File.exists?("index.json") do - existing_data = File.read!("index.json") |> Jason.decode!() + pack_file = "#{name}.json" + + if File.exists?(pack_file) do + existing_data = File.read!(pack_file) |> Jason.decode!() File.write!( - "index.json", + pack_file, Jason.encode!( Map.merge( existing_data, @@ -211,11 +229,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) ) - IO.puts("index.json file has been update with the #{name} pack") + IO.puts("#{pack_file} has been updated with the #{name} pack") else - File.write!("index.json", Jason.encode!(pack_json, pretty: true)) + File.write!(pack_file, Jason.encode!(pack_json, pretty: true)) - IO.puts("index.json has been created with the #{name} pack") + IO.puts("#{pack_file} has been created with the #{name} pack") end end -- cgit v1.2.3 From e67cde0ed6b55450b5f309f9ed86f7f8e2a1e73f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Apr 2020 13:46:34 +0200 Subject: Transmogrifier: Refactoring / Renaming. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a4b385cd5..455f51fe0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -617,9 +617,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)}, cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)), :ok <- ObjectValidator.fetch_actor_and_object(cast_data), - {_, {:ok, cast_data}} <- {:maybe_add_context, maybe_add_context_from_object(cast_data)}, + {_, {:ok, cast_data}} <- {:ensure_context_presence, ensure_context_presence(cast_data)}, {_, {:ok, cast_data}} <- - {:maybe_add_recipients, maybe_add_recipients_from_object(cast_data)}, + {:ensure_recipients_presence, ensure_recipients_presence(cast_data)}, {_, {:ok, activity, _meta}} <- {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do {:ok, activity} @@ -1251,10 +1251,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_user_object(data), do: maybe_fix_user_url(data) - defp maybe_add_context_from_object(%{"context" => context} = data) when is_binary(context), + defp ensure_context_presence(%{"context" => context} = data) when is_binary(context), do: {:ok, data} - defp maybe_add_context_from_object(%{"object" => object} = data) when is_binary(object) do + defp ensure_context_presence(%{"object" => object} = data) when is_binary(object) do with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do {:ok, Map.put(data, "context", context)} else @@ -1263,14 +1263,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - defp maybe_add_context_from_object(_) do + defp ensure_context_presence(_) do {:error, :no_context} end - defp maybe_add_recipients_from_object(%{"to" => [_ | _], "cc" => [_ | _]} = data), + defp ensure_recipients_presence(%{"to" => [_ | _], "cc" => [_ | _]} = data), do: {:ok, data} - defp maybe_add_recipients_from_object(%{"object" => object} = data) do + defp ensure_recipients_presence(%{"object" => object} = data) do case Object.normalize(object) do %{data: %{"actor" => actor}} -> data = @@ -1288,7 +1288,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - defp maybe_add_recipients_from_object(_) do + defp ensure_recipients_presence(_) do {:error, :no_object} end end -- cgit v1.2.3 From 772bc258cde11b3203ad9420f69321ccd56db91a Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Apr 2020 13:53:24 +0200 Subject: ObjectID Validator: Refactor. --- .../activity_pub/object_validators/types/object_id.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex index 8e70effe4..ee10be0b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -4,14 +4,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do def type, do: :string def cast(object) when is_binary(object) do - with %URI{ - scheme: scheme, - host: host - } - when scheme in ["https", "http"] and not is_nil(host) <- - URI.parse(object) do - {:ok, object} - else + # Host has to be present and scheme has to be an http scheme (for now) + case URI.parse(object) do + %URI{host: nil} -> + :error + + %URI{scheme: scheme} when scheme in ["https", "http"] -> + {:ok, object} + _ -> :error end -- cgit v1.2.3 From 03eebabe8e5b2e3f96f6ffe51a6f063a42f6a5d2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 3 Apr 2020 22:52:25 +0400 Subject: Add Pleroma.Web.ApiSpec.Helpers --- lib/pleroma/web/api_spec/helpers.ex | 27 ++++++++++++++++++++++ .../web/api_spec/operations/app_operation.ex | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/api_spec/helpers.ex (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex new file mode 100644 index 000000000..35cf4c0d8 --- /dev/null +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Helpers do + def request_body(description, schema_ref, opts \\ []) do + media_types = ["application/json", "multipart/form-data"] + + content = + media_types + |> Enum.map(fn type -> + {type, + %OpenApiSpex.MediaType{ + schema: schema_ref, + example: opts[:example], + examples: opts[:examples] + }} + end) + |> Enum.into(%{}) + + %OpenApiSpex.RequestBody{ + description: description, + content: content, + required: opts[:required] || false + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex index 41d56693a..26d8dbd42 100644 --- a/lib/pleroma/web/api_spec/operations/app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.AppOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Helpers alias Pleroma.Web.ApiSpec.Schemas.AppCreateRequest alias Pleroma.Web.ApiSpec.Schemas.AppCreateResponse @@ -21,8 +22,7 @@ defmodule Pleroma.Web.ApiSpec.AppOperation do summary: "Create an application", description: "Create a new application to obtain OAuth2 credentials", operationId: "AppController.create", - requestBody: - Operation.request_body("Parameters", "application/json", AppCreateRequest, required: true), + requestBody: Helpers.request_body("Parameters", AppCreateRequest, required: true), responses: %{ 200 => Operation.response("App", "application/json", AppCreateResponse), 422 => -- cgit v1.2.3 From 5739c498c029914c446656244cdd213a3e358fec Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 8 Apr 2020 18:46:01 +0300 Subject: fix for gun connections pool --- lib/pleroma/gun/conn.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 20823a765..cd25a2e74 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -49,8 +49,10 @@ defmodule Pleroma.Gun.Conn do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + max_connections = pool_opts[:max_connections] || 250 + conn_pid = - if Connections.count(name) < opts[:max_connection] do + if Connections.count(name) < max_connections do do_open(uri, opts) else close_least_used_and_do_open(name, uri, opts) -- cgit v1.2.3 From d067eaa7b3bb76e7fc5ae019d6e00510b657171d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 8 Apr 2020 22:58:31 +0300 Subject: formatter.ex: Use Phoenix.HTML for mention/hashtag generation Unlike concatenating strings, this makes sure everything is escaped. Tests had to be changed because Phoenix.HTML runs attributes through Enum.sort before generation for whatever reason. --- lib/pleroma/formatter.ex | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index e2a658cb3..c44e7fc8b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -35,9 +35,19 @@ defmodule Pleroma.Formatter do nickname_text = get_nickname_text(nickname, opts) link = - ~s(@#{ - nickname_text - }) + Phoenix.HTML.Tag.content_tag( + :span, + Phoenix.HTML.Tag.content_tag( + :a, + ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)], + "data-user": id, + class: "u-url mention", + href: ap_id, + rel: "ugc" + ), + class: "h-card" + ) + |> Phoenix.HTML.safe_to_string() {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}} @@ -49,7 +59,15 @@ defmodule Pleroma.Formatter do def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do tag = String.downcase(tag) url = "#{Pleroma.Web.base_url()}/tag/#{tag}" - link = ~s(#{tag_text}) + + link = + Phoenix.HTML.Tag.content_tag(:a, tag_text, + class: "hashtag", + "data-tag": tag, + href: url, + rel: "tag ugc" + ) + |> Phoenix.HTML.safe_to_string() {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}} end -- cgit v1.2.3 From c401b00c7885823744183dbd077db9239585d20d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Apr 2020 04:36:39 +0200 Subject: ObjectValidators.Types.ObjectID: Fix when URI.parse returns %URL{host: ""} --- .../web/activity_pub/object_validators/types/object_id.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex index ee10be0b0..f6e749b33 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -6,14 +6,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do def cast(object) when is_binary(object) do # Host has to be present and scheme has to be an http scheme (for now) case URI.parse(object) do - %URI{host: nil} -> - :error - - %URI{scheme: scheme} when scheme in ["https", "http"] -> - {:ok, object} - - _ -> - :error + %URI{host: nil} -> :error + %URI{host: ""} -> :error + %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, object} + _ -> :error end end -- cgit v1.2.3 From 4c60fdcbb1ab06183b8e300cbbb84d70ecd3e25b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:31 +0000 Subject: Apply suggestion to lib/pleroma/web/admin_api/admin_api_controller.ex --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 7b442f6e1..a66db68f3 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -592,7 +592,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do {:registrations_open, _} -> errors( conn, - {:error, "To send invites you need set `registrations_open` option to false."} + {:error, "To send invites you need to set the `registrations_open` option to false."} ) {:invites_enabled, _} -> -- cgit v1.2.3 From 1cf0d5ab0d579ee4a1a779c308fedb0ab8ec3884 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:36 +0000 Subject: Apply suggestion to lib/pleroma/web/admin_api/admin_api_controller.ex --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index a66db68f3..09959b3bf 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -598,7 +598,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do {:invites_enabled, _} -> errors( conn, - {:error, "To send invites you need set `invites_enabled` option to true."} + {:error, "To send invites you need set to set the `invites_enabled` option to true."} ) end end -- cgit v1.2.3 From f20a19de853e8834f7774ee0098a14213bc7427f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 13:28:54 +0300 Subject: typo fix --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 09959b3bf..fdbd24acb 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -598,7 +598,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do {:invites_enabled, _} -> errors( conn, - {:error, "To send invites you need set to set the `invites_enabled` option to true."} + {:error, "To send invites you need to set the `invites_enabled` option to true."} ) end end -- cgit v1.2.3 From d545b883eb3c5b79b89a49ccaf9256c31b401145 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 9 Apr 2020 17:08:43 +0400 Subject: Add `/api/v1/notifications/:id/dismiss` endpoint --- lib/pleroma/web/mastodon_api/controllers/notification_controller.ex | 3 ++- lib/pleroma/web/router.ex | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 0c9218454..a6b4096ec 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -66,7 +66,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do json(conn, %{}) end - # POST /api/v1/notifications/dismiss + # POST /api/v1/notifications/:id/dismiss + # POST /api/v1/notifications/dismiss (deprecated) def dismiss(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do with {:ok, _notif} <- Notification.dismiss(user, id) do json(conn, %{}) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3ecd59cd1..5f5ec1c81 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -352,9 +352,11 @@ defmodule Pleroma.Web.Router do get("/notifications", NotificationController, :index) get("/notifications/:id", NotificationController, :show) + post("/notifications/:id/dismiss", NotificationController, :dismiss) post("/notifications/clear", NotificationController, :clear) - post("/notifications/dismiss", NotificationController, :dismiss) delete("/notifications/destroy_multiple", NotificationController, :destroy_multiple) + # Deprecated: was removed in Mastodon v3, use `/notifications/:id/dismiss` instead + post("/notifications/dismiss", NotificationController, :dismiss) get("/scheduled_statuses", ScheduledActivityController, :index) get("/scheduled_statuses/:id", ScheduledActivityController, :show) -- cgit v1.2.3