From f6d09fafee83514889bbcf6531e0bc01e33b0b16 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 1 Mar 2020 09:48:32 +0100 Subject: Add support for remote favicons --- lib/pleroma/user.ex | 30 ++++++++++++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 3 ++- 2 files changed, 32 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e98332744..25ea112a2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2253,4 +2253,34 @@ defmodule Pleroma.User do |> Map.put(:bio, HTML.filter_tags(user.bio, filter)) |> Map.put(:fields, fields) end + + def get_cached_favicon(%User{} = user) do + key = "favicon:#{user.ap_id}" + Cachex.fetch!(:user_cache, key, fn _ -> get_favicon(user) end) + end + + def get_cached_favicon(_user) do + nil + end + + def get_favicon(user) do + try do + with url <- user.ap_id, + true <- is_binary(url), + {:ok, %Tesla.Env{body: html}} <- Pleroma.HTTP.get(url), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon_url <- URI.merge(URI.parse(url), favicon_rel) |> to_string(), + true <- is_binary(favicon_url) do + favicon_url + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index a6e64b4ab..efe835e3c 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -245,7 +245,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do hide_favorites: user.hide_favorites, relationship: relationship, skip_thread_containment: user.skip_thread_containment, - background_image: image_url(user.background) |> MediaProxy.url() + background_image: image_url(user.background) |> MediaProxy.url(), + favicon: User.get_cached_favicon(user) |> MediaProxy.url() } } |> maybe_put_role(user, opts[:for]) -- cgit v1.2.3 From 6a679d80c9030afa8327377928f8ac2fcf1a4a0e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 2 Mar 2020 05:38:25 +0100 Subject: Move get_favicon to Pleroma.Instances, use / --- lib/pleroma/application.ex | 1 + lib/pleroma/instances.ex | 28 ++++++++++++++++++++ lib/pleroma/user.ex | 30 ---------------------- lib/pleroma/web/mastodon_api/views/account_view.ex | 11 +++++++- 4 files changed, 39 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9615af122..c7fc95f75 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -150,6 +150,7 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("instances", default_ttl: 25_000, ttl_interval: 1000, limit: 2500) ] end diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 557e8decf..c9b1ed4ce 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -37,4 +37,32 @@ defmodule Pleroma.Instances do url_or_host end end + + def get_cached_favicon(instance_url) when is_binary(instance_url) do + Cachex.fetch!(:instances_cache, instance_url, fn _ -> get_favicon(instance_url) end) + end + + def get_cached_favicon(_instance_url) do + nil + end + + def get_favicon(instance_url) when is_binary(instance_url) do + try do + with {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(instance_url, [{:Accept, "text/html"}]), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon_url <- URI.merge(URI.parse(instance_url), favicon_rel) |> to_string(), + true <- is_binary(favicon_url) do + favicon_url + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 25ea112a2..e98332744 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2253,34 +2253,4 @@ defmodule Pleroma.User do |> Map.put(:bio, HTML.filter_tags(user.bio, filter)) |> Map.put(:fields, fields) end - - def get_cached_favicon(%User{} = user) do - key = "favicon:#{user.ap_id}" - Cachex.fetch!(:user_cache, key, fn _ -> get_favicon(user) end) - end - - def get_cached_favicon(_user) do - nil - end - - def get_favicon(user) do - try do - with url <- user.ap_id, - true <- is_binary(url), - {:ok, %Tesla.Env{body: html}} <- Pleroma.HTTP.get(url), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon_url <- URI.merge(URI.parse(url), favicon_rel) |> to_string(), - true <- is_binary(favicon_url) do - favicon_url - else - _ -> nil - end - rescue - _ -> nil - end - end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index efe835e3c..3ee50dfd0 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -204,6 +204,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end + favicon = + user + |> Map.get(:ap_id, "") + |> URI.parse() + |> URI.merge("/") + |> to_string() + |> Pleroma.Instances.get_cached_favicon() + |> MediaProxy.url() + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -246,7 +255,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do relationship: relationship, skip_thread_containment: user.skip_thread_containment, background_image: image_url(user.background) |> MediaProxy.url(), - favicon: User.get_cached_favicon(user) |> MediaProxy.url() + favicon: favicon } } |> maybe_put_role(user, opts[:for]) -- cgit v1.2.3 From 013e2c505786dff311bcc8bf23631d6a1a1636ef Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 7 Jul 2020 11:13:38 +0200 Subject: Use instances table instead of Cachex --- lib/pleroma/application.ex | 1 - lib/pleroma/instances.ex | 28 ----------- lib/pleroma/instances/instance.ex | 55 +++++++++++++++++++++- lib/pleroma/web/mastodon_api/views/account_view.ex | 3 +- 4 files changed, 55 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c7fc95f75..9615af122 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -150,7 +150,6 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) - build_cachex("instances", default_ttl: 25_000, ttl_interval: 1000, limit: 2500) ] end diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index c9b1ed4ce..557e8decf 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -37,32 +37,4 @@ defmodule Pleroma.Instances do url_or_host end end - - def get_cached_favicon(instance_url) when is_binary(instance_url) do - Cachex.fetch!(:instances_cache, instance_url, fn _ -> get_favicon(instance_url) end) - end - - def get_cached_favicon(_instance_url) do - nil - end - - def get_favicon(instance_url) when is_binary(instance_url) do - try do - with {:ok, %Tesla.Env{body: html}} <- - Pleroma.HTTP.get(instance_url, [{:Accept, "text/html"}]), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon_url <- URI.merge(URI.parse(instance_url), favicon_rel) |> to_string(), - true <- is_binary(favicon_url) do - favicon_url - else - _ -> nil - end - rescue - _ -> nil - end - end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 74458c09a..b97e229e5 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Instances.Instance do schema "instances" do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) + field(:favicon, :string) + field(:favicon_updated_at, :naive_datetime) timestamps() end @@ -25,7 +27,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since]) + |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -120,4 +122,55 @@ defmodule Pleroma.Instances.Instance do end defp parse_datetime(datetime), do: datetime + + def get_or_update_favicon(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() + + if existing_record && existing_record.favicon && + NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + existing_record.favicon + else + favicon = scrape_favicon(instance_uri) + + cond do + is_binary(favicon) && existing_record -> + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + + favicon + + is_binary(favicon) -> + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() + + favicon + + true -> + nil + end + end + end + + defp scrape_favicon(%URI{} = instance_uri) do + try do + with {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), + true <- is_binary(favicon) do + favicon + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 3ee50dfd0..db5739254 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -209,8 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> Map.get(:ap_id, "") |> URI.parse() |> URI.merge("/") - |> to_string() - |> Pleroma.Instances.get_cached_favicon() + |> Pleroma.Instances.Instance.get_or_update_favicon() |> MediaProxy.url() %{ -- cgit v1.2.3 From 8c9df2d2e6a5a3639f6b411cd3e9b57248a0650c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 7 Jul 2020 12:07:30 +0200 Subject: instance: Prevent loop of updates --- lib/pleroma/instances/instance.ex | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index b97e229e5..a1f935232 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -127,30 +127,23 @@ defmodule Pleroma.Instances.Instance do existing_record = Repo.get_by(Instance, %{host: host}) now = NaiveDateTime.utc_now() - if existing_record && existing_record.favicon && + if existing_record && existing_record.favicon_updated_at && NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do existing_record.favicon else favicon = scrape_favicon(instance_uri) - cond do - is_binary(favicon) && existing_record -> - existing_record - |> changeset(%{favicon: favicon, favicon_updated_at: now}) - |> Repo.update() - - favicon - - is_binary(favicon) -> - %Instance{} - |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) - |> Repo.insert() - - favicon - - true -> - nil + if existing_record do + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() end + + favicon end end -- cgit v1.2.3 From 312fc55f14e1b7f88ec43b72c577bf5df595beac Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 8 Jul 2020 05:56:24 +0200 Subject: Add [:instances_favicons, :enabled] setting, defaults to false --- lib/pleroma/web/mastodon_api/views/account_view.ex | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index db5739254..2feba4778 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -205,12 +205,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end favicon = - user - |> Map.get(:ap_id, "") - |> URI.parse() - |> URI.merge("/") - |> Pleroma.Instances.Instance.get_or_update_favicon() - |> MediaProxy.url() + if Pleroma.Config.get([:instances_favicons, :enabled]) do + user + |> Map.get(:ap_id, "") + |> URI.parse() + |> URI.merge("/") + |> Pleroma.Instances.Instance.get_or_update_favicon() + |> MediaProxy.url() + else + nil + end %{ id: to_string(user.id), -- cgit v1.2.3 From 31fef95e35d3cbc2d65c76a57ba8f4047809ce1b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 8 Jul 2020 06:09:39 +0200 Subject: Add changelog and documentation --- lib/pleroma/web/api_spec/schemas/account.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 84f18f1b6..e6f163cb7 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -102,6 +102,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, description: "A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`" + }, + favicon: %Schema{ + type: :string, + format: :uri, + nullable: true, + description: "Favicon image of the user's instance" } } }, -- cgit v1.2.3