From e5c6bf3673a8361d1417eba1ccc44edec7658ac4 Mon Sep 17 00:00:00 2001 From: shadowfacts Date: Sat, 7 Sep 2019 19:50:45 +0000 Subject: Mastodon API: URI encode hashtag name in generated URLs Otherwise hashtags with word characters other than those allowed in URLs (e.g. Japanese characters) produce hashtag URLs that are invalid. --- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b91..708b8c2fd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -499,7 +499,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do object_tags = for tag when is_binary(tag) <- object_tags, do: tag Enum.reduce(object_tags, [], fn tag, tags -> - tags ++ [%{name: tag, url: "/tag/#{tag}"}] + tags ++ [%{name: tag, url: "/tag/#{URI.encode(tag)}"}] end) end -- cgit v1.2.3 From b40b10b53d00d13f24b5667acc02b1642abc6ec4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 16:23:03 +0700 Subject: Add an endpoint to get multiple statuses by IDs --- lib/pleroma/activity.ex | 7 +++++++ .../mastodon_api/controllers/mastodon_api_controller.ex | 14 ++++++++++++++ lib/pleroma/web/router.ex | 1 + 3 files changed, 22 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6a51d4cf3..44f1e3011 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -173,6 +173,13 @@ defmodule Pleroma.Activity do |> Repo.one() end + def all_by_ids_with_object(ids) do + Activity + |> where([a], a.id in ^ids) + |> with_preloaded_object() + |> Repo.all() + end + def by_object_ap_id(ap_id) do from( activity in Activity, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 8dfad7a54..c54462bb3 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -427,6 +427,20 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> render("index.json", %{activities: activities, for: user, as: :activity}) end + def get_statuses(%{assigns: %{user: user}} = conn, %{"ids" => ids}) do + limit = 100 + + activities = + ids + |> Enum.take(limit) + |> Activity.all_by_ids_with_object() + |> Enum.filter(&Visibility.visible_for_user?(&1, user)) + + conn + |> put_view(StatusView) + |> render("index.json", activities: activities, for: user, as: :activity) + end + def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index cfb973f53..7cd59acb2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -443,6 +443,7 @@ defmodule Pleroma.Web.Router do get("/timelines/tag/:tag", MastodonAPIController, :hashtag_timeline) get("/timelines/list/:list_id", MastodonAPIController, :list_timeline) + get("/statuses", MastodonAPIController, :get_statuses) get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) -- cgit v1.2.3 From 43f02dfe38547e07fb189aab78539af9e02302b3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 22:01:45 +0300 Subject: Revert "Parallelize template rendering" This reverts commit 1ad71592adb47762287aec8c36d0fca565c38362. Since it had no limit on the number on concurrent processes it OOM killed instances while rendering hellthreads. When I tried introducing a concurrency limit with Task.async_stream/manual folds it lead to about 3 times worse performance on threads larger than 1000 activities (we are talking 30s vs 1.2 minutes), I think this is not worth the about 1.5 times performance increase on smaller threads when using it. --- lib/mix/tasks/pleroma/benchmark.ex | 38 ++++++----------------- lib/pleroma/web/mastodon_api/views/status_view.ex | 4 +-- lib/pleroma/web/web.ex | 18 ++--------- 3 files changed, 12 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 4cc634727..a45940bf3 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -37,37 +37,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) - |> Map.put("limit", 80) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - inputs = %{ - "One activity" => Enum.take_random(activities, 1), - "Ten activities" => Enum.take_random(activities, 10), - "Twenty activities" => Enum.take_random(activities, 20), - "Forty activities" => Enum.take_random(activities, 40), - "Eighty activities" => Enum.take_random(activities, 80) - } - - Benchee.run( - %{ - "Parallel rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end, - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity, - parallel: false - }) - end - }, - inputs: inputs - ) + Benchee.run(%{ + "render_timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b91..b6a3431f9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -73,14 +73,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) - parallel = unless is_nil(opts[:parallel]), do: opts[:parallel], else: true opts.activities |> safe_render_many( StatusView, "status.json", - Map.put(opts, :replied_to_activities, replied_to_activities), - parallel + Map.put(opts, :replied_to_activities, replied_to_activities) ) end diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index bfb6c7287..687346554 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -66,23 +66,9 @@ defmodule Pleroma.Web do end @doc """ - Same as `render_many/4` but wrapped in rescue block and parallelized (unless disabled by passing false as a fifth argument). + Same as `render_many/4` but wrapped in rescue block. """ - def safe_render_many(collection, view, template, assigns \\ %{}, parallel \\ true) - - def safe_render_many(collection, view, template, assigns, true) do - Enum.map(collection, fn resource -> - Task.async(fn -> - as = Map.get(assigns, :as) || view.__resource__ - assigns = Map.put(assigns, as, resource) - safe_render(view, template, assigns) - end) - end) - |> Enum.map(&Task.await(&1, :infinity)) - |> Enum.filter(& &1) - end - - def safe_render_many(collection, view, template, assigns, false) do + def safe_render_many(collection, view, template, assigns \\ %{}) do Enum.map(collection, fn resource -> as = Map.get(assigns, :as) || view.__resource__ assigns = Map.put(assigns, as, resource) -- cgit v1.2.3 From 5a76d5d2391f0fa6b4c58bf190b0cdbfff96014f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 23:08:15 +0300 Subject: Add extended benchmark --- lib/mix/tasks/pleroma/benchmark.ex | 40 ++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a45940bf3..84dccf7f3 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do }) end - def run(["render_timeline", nickname]) do + def run(["render_timeline", nickname | _] = args) do start_pleroma() user = Pleroma.User.get_by_nickname(nickname) @@ -37,17 +37,41 @@ defmodule Mix.Tasks.Pleroma.Benchmark do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) + |> Map.put("limit", 4096) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - Benchee.run(%{ - "render_timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity + inputs = %{ + "1 activity" => Enum.take_random(activities, 1), + "10 activities" => Enum.take_random(activities, 10), + "20 activities" => Enum.take_random(activities, 20), + "40 activities" => Enum.take_random(activities, 40), + "80 activities" => Enum.take_random(activities, 80) + } + + inputs = + if Enum.at(args, 2) == "extended" do + Map.merge(inputs, %{ + "200 activities" => Enum.take_random(activities, 200), + "500 activities" => Enum.take_random(activities, 500), + "2000 activities" => Enum.take_random(activities, 2000), + "4096 activities" => Enum.take_random(activities, 4096) }) + else + inputs end - }) + + Benchee.run( + %{ + "Standart rendering" => fn activities -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }, + inputs: inputs + ) end end -- cgit v1.2.3 From 56828abf6de81a52079b26cf899b91fdd822f2ce Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 11 Sep 2019 23:04:01 +0300 Subject: Use Jason for rendering responses Although Jason readme says Phoenix 1.4+ already does it by default, [it actually does it only for new projects](https://github.com/phoenixframework/phoenix/blob/3bfb9f6e900c9a2e31cb95736e2cb5bdad329b61/lib/phoenix.ex#L58-L59) --- lib/healthcheck.ex | 64 --------------------------------------------- lib/pleroma/healthcheck.ex | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 64 deletions(-) delete mode 100644 lib/healthcheck.ex create mode 100644 lib/pleroma/healthcheck.ex (limited to 'lib') diff --git a/lib/healthcheck.ex b/lib/healthcheck.ex deleted file mode 100644 index f97d14432..000000000 --- a/lib/healthcheck.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Healthcheck do - @moduledoc """ - Module collects metrics about app and assign healthy status. - """ - alias Pleroma.Healthcheck - alias Pleroma.Repo - - defstruct pool_size: 0, - active: 0, - idle: 0, - memory_used: 0, - healthy: true - - @type t :: %__MODULE__{ - pool_size: non_neg_integer(), - active: non_neg_integer(), - idle: non_neg_integer(), - memory_used: number(), - healthy: boolean() - } - - @spec system_info() :: t() - def system_info do - %Healthcheck{ - memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) - } - |> assign_db_info() - |> check_health() - end - - defp assign_db_info(healthcheck) do - database = Pleroma.Config.get([Repo, :database]) - - query = - "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;" - - result = Repo.query!(query) - pool_size = Pleroma.Config.get([Repo, :pool_size]) - - db_info = - Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states -> - if state == "active" do - Map.put(states, :active, states.active + cnt) - else - Map.put(states, :idle, states.idle + cnt) - end - end) - |> Map.put(:pool_size, pool_size) - - Map.merge(healthcheck, db_info) - end - - @spec check_health(Healthcheck.t()) :: Healthcheck.t() - def check_health(%{pool_size: pool_size, active: active} = check) - when active >= pool_size do - %{check | healthy: false} - end - - def check_health(check), do: check -end diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex new file mode 100644 index 000000000..977b78c26 --- /dev/null +++ b/lib/pleroma/healthcheck.ex @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Healthcheck do + @moduledoc """ + Module collects metrics about app and assign healthy status. + """ + alias Pleroma.Healthcheck + alias Pleroma.Repo + + @derive Jason.Encoder + defstruct pool_size: 0, + active: 0, + idle: 0, + memory_used: 0, + healthy: true + + @type t :: %__MODULE__{ + pool_size: non_neg_integer(), + active: non_neg_integer(), + idle: non_neg_integer(), + memory_used: number(), + healthy: boolean() + } + + @spec system_info() :: t() + def system_info do + %Healthcheck{ + memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) + } + |> assign_db_info() + |> check_health() + end + + defp assign_db_info(healthcheck) do + database = Pleroma.Config.get([Repo, :database]) + + query = + "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;" + + result = Repo.query!(query) + pool_size = Pleroma.Config.get([Repo, :pool_size]) + + db_info = + Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states -> + if state == "active" do + Map.put(states, :active, states.active + cnt) + else + Map.put(states, :idle, states.idle + cnt) + end + end) + |> Map.put(:pool_size, pool_size) + + Map.merge(healthcheck, db_info) + end + + @spec check_health(Healthcheck.t()) :: Healthcheck.t() + def check_health(%{pool_size: pool_size, active: active} = check) + when active >= pool_size do + %{check | healthy: false} + end + + def check_health(check), do: check +end -- cgit v1.2.3