diff options
Diffstat (limited to 'lib')
88 files changed, 1512 insertions, 986 deletions
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 9f0bf6ecb..074492a46 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -24,8 +24,10 @@ defmodule Mix.Pleroma do Application.put_env(:logger, :console, level: :debug) end + adapter = Application.get_env(:tesla, :adapter) + apps = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if adapter == Tesla.Adapter.Gun do [:gun | @apps] else [:hackney | @apps] @@ -33,11 +35,14 @@ defmodule Mix.Pleroma do Enum.each(apps, &Application.ensure_all_started/1) - children = [ - Pleroma.Repo, - {Pleroma.Config.TransferTask, false}, - Pleroma.Web.Endpoint - ] + children = + [ + Pleroma.Repo, + {Pleroma.Config.TransferTask, false}, + Pleroma.Web.Endpoint, + {Oban, Pleroma.Config.get(Oban)} + ] ++ + http_children(adapter) cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -115,4 +120,11 @@ defmodule Mix.Pleroma do def escape_sh_path(path) do ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(') end + + defp http_children(Tesla.Adapter.Gun) do + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] + end + + defp http_children(_), do: [] end diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex index 7d65f0587..00f5ba7bf 100644 --- a/lib/mix/tasks/pleroma/notification_settings.ex +++ b/lib/mix/tasks/pleroma/notification_settings.ex @@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do @moduledoc """ Example: - > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588" # set false only for parallel588 user - > mix pleroma.notification_settings --privacy-option=true # set true for all users + > mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588" # set false only for parallel588 user + > mix pleroma.notification_settings --hide-notification-contents=true # set true for all users """ @@ -19,16 +19,16 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do OptionParser.parse( args, strict: [ - privacy_option: :boolean, + hide_notification_contents: :boolean, email_users: :string, nickname_users: :string ] ) - privacy_option = Keyword.get(options, :privacy_option) + hide_notification_contents = Keyword.get(options, :hide_notification_contents) - if not is_nil(privacy_option) do - privacy_option + if not is_nil(hide_notification_contents) do + hide_notification_contents |> build_query(options) |> Pleroma.Repo.update_all([]) end @@ -36,15 +36,15 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do shell_info("Done") end - defp build_query(privacy_option, options) do + defp build_query(hide_notification_contents, options) do query = from(u in Pleroma.User, update: [ set: [ notification_settings: fragment( - "jsonb_set(notification_settings, '{privacy_option}', ?)", - ^privacy_option + "jsonb_set(notification_settings, '{hide_notification_contents}', ?)", + ^hide_notification_contents ) ] ] diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex new file mode 100644 index 000000000..9da74ffcf --- /dev/null +++ b/lib/mix/tasks/pleroma/release_env.ex @@ -0,0 +1,76 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.ReleaseEnv do + use Mix.Task + import Mix.Pleroma + + @shortdoc "Generate Pleroma environment file." + @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md") + + def run(["gen" | rest]) do + {options, [], []} = + OptionParser.parse( + rest, + strict: [ + force: :boolean, + path: :string + ], + aliases: [ + p: :path, + f: :force + ] + ) + + file_path = + get_option( + options, + :path, + "Environment file path", + "./config/pleroma.env" + ) + + env_path = Path.expand(file_path) + + proceed? = + if File.exists?(env_path) do + get_option( + options, + :force, + "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)", + "n" + ) === "y" + else + true + end + + if proceed? do + case do_generate(env_path) do + {:error, reason} -> + shell_error( + File.Error.message(%{action: "write to file", reason: reason, path: env_path}) + ) + + _ -> + shell_info("\nThe file generated: #{env_path}.\n") + + shell_info(""" + WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable. + Example: + chmod 0444 #{file_path} + chattr +i #{file_path} + """) + end + else + shell_info("\nThe file is exist. #{env_path}.\n") + end + end + + def do_generate(path) do + content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}" + + File.mkdir_p!(Path.dirname(path)) + File.write(path, content) + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3282c6882..0ffb55358 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,6 +39,7 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() Config.DeprecationWarnings.warn() @@ -223,9 +224,7 @@ defmodule Pleroma.Application do # start hackney and gun pools in tests defp http_children(_, :test) do - hackney_options = Config.get([:hackney_pools, :federation]) - hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) - [hackney_pool, Pleroma.Pool.Supervisor] + http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil) end defp http_children(Tesla.Adapter.Hackney, _) do @@ -244,7 +243,10 @@ defmodule Pleroma.Application do end end - defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] + defp http_children(Tesla.Adapter.Gun, _) do + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] + end defp http_children(_, _), do: [] end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 88575a498..16f62b6f5 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -16,7 +16,9 @@ defmodule Pleroma.ApplicationRequirements do @spec verify!() :: :ok | VerifyError.t() def verify! do :ok + |> check_confirmation_accounts! |> check_migrations_applied!() + |> check_welcome_message_config!() |> check_rum!() |> handle_result() end @@ -24,6 +26,40 @@ defmodule Pleroma.ApplicationRequirements do defp handle_result(:ok), do: :ok defp handle_result({:error, message}), do: raise(VerifyError, message: message) + defp check_welcome_message_config!(:ok) do + if Pleroma.Config.get([:welcome, :email, :enabled], false) and + not Pleroma.Emails.Mailer.enabled?() do + Logger.error(""" + To send welcome email do you need to enable mail. + \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true + """) + + {:error, "The mail disabled."} + else + :ok + end + end + + defp check_welcome_message_config!(result), do: result + + # Checks account confirmation email + # + def check_confirmation_accounts!(:ok) do + if Pleroma.Config.get([:instance, :account_activation_required]) && + not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + Logger.error( + "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer." + ) + + {:error, + "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} + else + :ok + end + end + + def check_confirmation_accounts!(result), do: result + # Checks for pending migrations. # def check_migrations_applied!(:ok) do diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 1a89d8895..e5b7811aa 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -156,7 +156,6 @@ defmodule Pleroma.ConfigDB do {:quack, :meta}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:auto_linker, :opts}, {:swarm, :node_blacklist}, {:logger, :backends} ] diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 026871c4f..0f52eb210 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -55,6 +55,24 @@ defmodule Pleroma.Config.DeprecationWarnings do mrf_user_allowlist() check_old_mrf_config() check_media_proxy_whitelist_config() + check_welcome_message_config() + end + + def check_welcome_message_config do + instance_config = Pleroma.Config.get([:instance]) + + use_old_config = + Keyword.has_key?(instance_config, :welcome_user_nickname) or + Keyword.has_key?(instance_config, :welcome_message) + + if use_old_config do + Logger.error(""" + !!!DEPRECATION WARNING!!! + Your config is using the old namespace for Welcome messages configuration. You need to change to the new namespace: + \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname` + \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message` + """) + end end def check_old_mrf_config do diff --git a/lib/pleroma/config/helpers.ex b/lib/pleroma/config/helpers.ex new file mode 100644 index 000000000..3dce40ea0 --- /dev/null +++ b/lib/pleroma/config/helpers.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Config.Helpers do + alias Pleroma.Config + + def instance_name, do: Config.get([:instance, :name]) + + defp instance_notify_email do + Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) + end + + def sender do + {instance_name(), instance_notify_email()} + end +end diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index aa0b2a66b..c27ad1065 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Emails.AdminEmail do import Swoosh.Email alias Pleroma.Config + alias Pleroma.HTML alias Pleroma.Web.Router.Helpers defp instance_config, do: Config.get(:instance) @@ -82,4 +83,18 @@ defmodule Pleroma.Emails.AdminEmail do |> subject("#{instance_name()} Report") |> html_body(html_body) end + + def new_unapproved_registration(to, account) do + html_body = """ + <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p> + <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote> + <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a> + """ + + new() + |> to({to.name, to.email}) + |> from({instance_name(), instance_notify_email()}) + |> subject("New account up for review on #{instance_name()} (@#{account.nickname})") + |> html_body(html_body) + end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index dfadc10b3..313533859 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -12,17 +12,22 @@ defmodule Pleroma.Emails.UserEmail do alias Pleroma.Web.Endpoint alias Pleroma.Web.Router - defp instance_name, do: Config.get([:instance, :name]) - - defp sender do - email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) - {instance_name(), email} - end + import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0] defp recipient(email, nil), do: email defp recipient(email, name), do: {name, email} defp recipient(%User{} = user), do: recipient(user.email, user.name) + @spec welcome(User.t(), map()) :: Swoosh.Email.t() + def welcome(user, opts \\ %{}) do + new() + |> to(recipient(user)) + |> from(Map.get(opts, :sender, sender())) + |> subject(Map.get(opts, :subject, "Welcome to #{instance_name()}!")) + |> html_body(Map.get(opts, :html, "Welcome to #{instance_name()}!")) + |> text_body(Map.get(opts, :text, "Welcome to #{instance_name()}!")) + end + def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 02a93a8dc..0c450eae4 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ - @auto_linker_config hashtag: true, - hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, - mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4, - scheme: true + defp linkify_opts do + Pleroma.Config.get(Pleroma.Formatter) ++ + [ + hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4 + ] + end def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do @@ -80,19 +84,19 @@ defmodule Pleroma.Formatter do @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do - options = options ++ @auto_linker_config + options = linkify_opts() ++ options if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) - {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options) {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} else acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options) {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end @@ -111,9 +115,9 @@ defmodule Pleroma.Formatter do if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) - AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options) + Linkify.link(mentions, options) <> Linkify.link(rest, options) else - AutoLinker.link(text, options) + Linkify.link(text, options) end end diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 3d56d50a9..e9f54c4c0 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -96,16 +96,18 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def response("/main/public") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"], "local_only" => true}) - |> render_activities + %{type: ["Create"], local_only: true} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Public Timeline!") <> posts <> ".\r\n" end def response("/main/all") do posts = - ActivityPub.fetch_public_activities(%{"type" => ["Create"]}) - |> render_activities + %{type: ["Create"]} + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Welcome to the Federated Timeline!") <> posts <> ".\r\n" end @@ -130,13 +132,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def response("/users/" <> nickname) do with %User{} = user <- User.get_cached_by_nickname(nickname) do params = %{ - "type" => ["Create"], - "actor_id" => user.ap_id + type: ["Create"], + actor_id: user.ap_id } activities = - ActivityPub.fetch_public_activities(params) - |> render_activities + params + |> ActivityPub.fetch_public_activities() + |> render_activities() info("Posts by #{user.nickname}") <> activities <> ".\r\n" else diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f51cd7db8..09be74392 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Gun.API do :tls_opts, :tcp_opts, :socks_opts, - :ws_opts + :ws_opts, + :supervise ] @impl Gun diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index cd25a2e74..a3f75a4bb 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -3,85 +3,33 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do - @moduledoc """ - Struct for gun connection data - """ alias Pleroma.Gun - alias Pleroma.Pool.Connections require Logger - @type gun_state :: :up | :down - @type conn_state :: :active | :idle - - @type t :: %__MODULE__{ - conn: pid(), - gun_state: gun_state(), - conn_state: conn_state(), - used_by: [pid()], - last_reference: pos_integer(), - crf: float(), - retries: pos_integer() - } - - defstruct conn: nil, - gun_state: :open, - conn_state: :init, - used_by: [], - last_reference: 0, - crf: 1, - retries: 0 - - @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil - def open(url, name, opts \\ []) - def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) - - def open(%URI{} = uri, name, opts) do + def open(%URI{} = uri, opts) do pool_opts = Pleroma.Config.get([:connections_pool], []) opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 1) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> Map.put_new(:supervise, false) |> maybe_add_tls_opts(uri) - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - max_connections = pool_opts[:max_connections] || 250 - - conn_pid = - if Connections.count(name) < max_connections do - do_open(uri, opts) - else - close_least_used_and_do_open(name, uri, opts) - end - - if is_pid(conn_pid) do - conn = %Pleroma.Gun.Conn{ - conn: conn_pid, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - } - - :ok = Gun.set_owner(conn_pid, Process.whereis(name)) - Connections.add_conn(name, key, conn) - end + do_open(uri, opts) end defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts - defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + defp maybe_add_tls_opts(opts, %URI{scheme: "https"}) do tls_opts = [ verify: :verify_peer, cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, - [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + log_level: :warning, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)] ] tls_opts = @@ -105,7 +53,7 @@ defmodule Pleroma.Gun.Conn do {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do - conn + {:ok, conn} else error -> Logger.warn( @@ -141,7 +89,7 @@ defmodule Pleroma.Gun.Conn do with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, conn} else error -> Logger.warn( @@ -155,11 +103,11 @@ defmodule Pleroma.Gun.Conn do end defp do_open(%URI{host: host, port: port} = uri, opts) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, conn} else error -> Logger.warn( @@ -171,7 +119,7 @@ defmodule Pleroma.Gun.Conn do end defp destination_opts(%URI{host: host, port: port}) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) %{host: host, port: port} end @@ -181,17 +129,6 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts - defp close_least_used_and_do_open(name, uri, opts) do - with [{key, conn} | _conns] <- Connections.get_unused_conns(name), - :ok <- Gun.close(conn.conn) do - Connections.remove_conn(name, key) - - do_open(uri, opts) - else - [] -> {:error, :pool_overflowed} - end - end - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do "#{scheme}://#{host}#{path}" end diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex new file mode 100644 index 000000000..49e9885bb --- /dev/null +++ b/lib/pleroma/gun/connection_pool.ex @@ -0,0 +1,79 @@ +defmodule Pleroma.Gun.ConnectionPool do + @registry __MODULE__ + + alias Pleroma.Gun.ConnectionPool.WorkerSupervisor + + def children do + [ + {Registry, keys: :unique, name: @registry}, + Pleroma.Gun.ConnectionPool.WorkerSupervisor + ] + end + + def get_conn(uri, opts) do + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + case Registry.lookup(@registry, key) do + # The key has already been registered, but connection is not up yet + [{worker_pid, nil}] -> + get_gun_pid_from_worker(worker_pid, true) + + [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> + GenServer.call(worker_pid, :add_client) + {:ok, gun_pid} + + [] -> + # :gun.set_owner fails in :connected state for whatevever reason, + # so we open the connection in the process directly and send it's pid back + # We trust gun to handle timeouts by itself + case WorkerSupervisor.start_worker([key, uri, opts, self()]) do + {:ok, worker_pid} -> + get_gun_pid_from_worker(worker_pid, false) + + {:error, {:already_started, worker_pid}} -> + get_gun_pid_from_worker(worker_pid, true) + + err -> + err + end + end + end + + defp get_gun_pid_from_worker(worker_pid, register) do + # GenServer.call will block the process for timeout length if + # the server crashes on startup (which will happen if gun fails to connect) + # so instead we use cast + monitor + + ref = Process.monitor(worker_pid) + if register, do: GenServer.cast(worker_pid, {:add_client, self()}) + + receive do + {:conn_pid, pid} -> + Process.demonitor(ref) + {:ok, pid} + + {:DOWN, ^ref, :process, ^worker_pid, reason} -> + case reason do + {:shutdown, error} -> error + _ -> {:error, reason} + end + end + end + + def release_conn(conn_pid) do + # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid -> + # worker_pid end) + query_result = + Registry.select(@registry, [ + {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + ]) + + case query_result do + [worker_pid] -> + GenServer.call(worker_pid, :remove_client) + + [] -> + :ok + end + end +end diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex new file mode 100644 index 000000000..cea800882 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -0,0 +1,85 @@ +defmodule Pleroma.Gun.ConnectionPool.Reclaimer do + use GenServer, restart: :temporary + + @registry Pleroma.Gun.ConnectionPool + + def start_monitor do + pid = + case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do + {:ok, pid} -> + pid + + {:error, {:already_registered, pid}} -> + pid + end + + {pid, Process.monitor(pid)} + end + + @impl true + def init(_) do + {:ok, nil, {:continue, :reclaim}} + end + + @impl true + def handle_continue(:reclaim, _) do + max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) + + reclaim_max = + [:connections_pool, :reclaim_multiplier] + |> Pleroma.Config.get() + |> Kernel.*(max_connections) + |> round + |> max(1) + + :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ + max_connections: max_connections, + reclaim_max: reclaim_max + }) + + # :ets.fun2ms( + # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> + # {worker_pid, crf, last_reference} end) + unused_conns = + Registry.select( + @registry, + [ + {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} + ] + ) + + case unused_conns do + [] -> + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + %{ + max_connections: max_connections + } + ) + + {:stop, :no_unused_conns, nil} + + unused_conns -> + reclaimed = + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + + reclaimed + |> Enum.each(fn {pid, _, _} -> + DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid) + end) + + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: Enum.count(reclaimed)}, + %{max_connections: max_connections} + ) + + {:stop, :normal, nil} + end + end +end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex new file mode 100644 index 000000000..fec9d0efa --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -0,0 +1,133 @@ +defmodule Pleroma.Gun.ConnectionPool.Worker do + alias Pleroma.Gun + use GenServer, restart: :temporary + + @registry Pleroma.Gun.ConnectionPool + + def start_link([key | _] = opts) do + GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}}) + end + + @impl true + def init([_key, _uri, _opts, _client_pid] = opts) do + {:ok, nil, {:continue, {:connect, opts}}} + end + + @impl true + def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do + with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + Process.link(conn_pid) do + time = :erlang.monotonic_time(:millisecond) + + {_, _} = + Registry.update_value(@registry, key, fn _ -> + {conn_pid, [client_pid], 1, time} + end) + + send(client_pid, {:conn_pid, conn_pid}) + + {:noreply, + %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, + :hibernate} + else + err -> + {:stop, {:shutdown, err}, nil} + end + end + + @impl true + def handle_cast({:add_client, client_pid}, state) do + case handle_call(:add_client, {client_pid, nil}, state) do + {:reply, conn_pid, state, :hibernate} -> + send(client_pid, {:conn_pid, conn_pid}) + {:noreply, state, :hibernate} + end + end + + @impl true + def handle_cast({:remove_client, client_pid}, state) do + case handle_call(:remove_client, {client_pid, nil}, state) do + {:reply, _, state, :hibernate} -> + {:noreply, state, :hibernate} + end + end + + @impl true + def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do + time = :erlang.monotonic_time(:millisecond) + + {{conn_pid, _, _, _}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} + end) + + state = + if state.timer != nil do + Process.cancel_timer(state[:timer]) + %{state | timer: nil} + else + state + end + + ref = Process.monitor(client_pid) + + state = put_in(state.client_monitors[client_pid], ref) + {:reply, conn_pid, state, :hibernate} + end + + @impl true + def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do + {{_conn_pid, used_by, _crf, _last_reference}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, List.delete(used_by, client_pid), crf, last_reference} + end) + + {ref, state} = pop_in(state.client_monitors[client_pid]) + Process.demonitor(ref) + + timer = + if used_by == [] do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + Process.send_after(self(), :idle_close, max_idle) + else + nil + end + + {:reply, :ok, %{state | timer: timer}, :hibernate} + end + + @impl true + def handle_info(:idle_close, state) do + # Gun monitors the owner process, and will close the connection automatically + # when it's terminated + {:stop, :normal, state} + end + + # Gracefully shutdown if the connection got closed without any streams left + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do + {:stop, :normal, state} + end + + # Otherwise, shutdown with an error + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do + {:stop, {:error, down_message}, state} + end + + @impl true + def handle_info({:DOWN, _ref, :process, pid, reason}, state) do + :telemetry.execute( + [:pleroma, :connection_pool, :client_death], + %{client_pid: pid, reason: reason}, + %{key: state.key} + ) + + handle_cast({:remove_client, pid}, state) + end + + # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 + defp crf(time_delta, prev_crf) do + 1 + :math.pow(0.5, 0.0001 * time_delta) * prev_crf + end +end diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex new file mode 100644 index 000000000..39615c956 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -0,0 +1,45 @@ +defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do + @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit" + + use DynamicSupervisor + + def start_link(opts) do + DynamicSupervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(_opts) do + DynamicSupervisor.init( + strategy: :one_for_one, + max_children: Pleroma.Config.get([:connections_pool, :max_connections]) + ) + end + + def start_worker(opts, retry \\ false) do + case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do + {:error, :max_children} -> + if retry or free_pool() == :error do + :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + {:error, :pool_full} + else + start_worker(opts, true) + end + + res -> + res + end + end + + defp free_pool do + wait_for_reclaimer_finish(Pleroma.Gun.ConnectionPool.Reclaimer.start_monitor()) + end + + defp wait_for_reclaimer_finish({pid, mon}) do + receive do + {:DOWN, ^mon, :process, ^pid, :no_unused_conns} -> + :error + + {:DOWN, ^mon, :process, ^pid, :normal} -> + :ok + end + end +end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 510722ff9..9ec3836b0 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -3,32 +3,30 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterHelper do - alias Pleroma.HTTP.Connection + @moduledoc """ + Configure Tesla.Client with default and customized adapter options. + """ + @defaults [pool: :federation] + + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | :inet.ip_address() + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + require Logger @type proxy :: {Connection.host(), pos_integer()} | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() - @callback after_request(keyword()) :: :ok - - @spec options(keyword(), URI.t()) :: keyword() - def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - maybe_add_proxy(opts, format_proxy(proxy)) - end - - @spec maybe_get_conn(URI.t(), keyword()) :: keyword() - def maybe_get_conn(_uri, opts), do: opts - - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok + @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()} @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil def format_proxy(nil), do: nil def format_proxy(proxy_url) do - case Connection.parse_proxy(proxy_url) do + case parse_proxy(proxy_url) do {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil @@ -38,4 +36,105 @@ defmodule Pleroma.HTTP.AdapterHelper do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @doc """ + Merge default connection & adapter options with received ones. + """ + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> put_timeout() + |> Keyword.merge(opts) + |> adapter_helper().options(uri) + end + + # For Hackney, this is the time a connection can stay idle in the pool. + # For Gun, this is the timeout to receive a message from Gun. + defp put_timeout(opts) do + {config_key, default} = + if adapter() == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout], 5_000)} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) + + Keyword.merge(opts, timeout: timeout) + end + + def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper.Default + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + :error -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | :inet.ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex new file mode 100644 index 000000000..e13441316 --- /dev/null +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -0,0 +1,14 @@ +defmodule Pleroma.HTTP.AdapterHelper.Default do + alias Pleroma.HTTP.AdapterHelper + + @behaviour Pleroma.HTTP.AdapterHelper + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) + end + + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} + def get_conn(_uri, opts), do: {:ok, opts} +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index ead7cdc6b..b4ff8306c 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,8 +5,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper + alias Pleroma.Gun.ConnectionPool alias Pleroma.HTTP.AdapterHelper - alias Pleroma.Pool.Connections require Logger @@ -14,7 +14,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 1, + retry: 0, retry_timeout: 1000, await_up_timeout: 5_000 ] @@ -31,16 +31,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.merge(config_opts) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(proxy) - |> maybe_get_conn(uri, incoming_opts) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts) do - if opts[:conn] && opts[:body_as] != :chunks do - Connections.checkout(opts[:conn], self(), :gun_connections) - end - - :ok + |> Keyword.merge(incoming_opts) end defp add_scheme_opts(opts, %{scheme: "http"}), do: opts @@ -48,30 +39,40 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:tls_opts, log_level: :warning) end - defp maybe_get_conn(adapter_opts, uri, incoming_opts) do - {receive_conn?, opts} = - adapter_opts - |> Keyword.merge(incoming_opts) - |> Keyword.pop(:receive_conn, true) - - if Connections.alive?(:gun_connections) and receive_conn? do - checkin_conn(uri, opts) - else - opts + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} + def get_conn(uri, opts) do + case ConnectionPool.get_conn(uri, opts) do + {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)} + err -> err end end - defp checkin_conn(uri, opts) do - case Connections.checkin(uri, :gun_connections) do - nil -> - Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) - opts + @prefix Pleroma.Gun.ConnectionPool + def limiter_setup do + wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait]) + retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries]) - conn when is_pid(conn) -> - Keyword.merge(opts, conn: conn, close_conn: false) - end + :pools + |> Pleroma.Config.get([]) + |> Enum.each(fn {name, opts} -> + max_running = Keyword.get(opts, :size, 50) + max_waiting = Keyword.get(opts, :max_waiting, 10) + + result = + ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting, + wait: wait, + max_retries: retries + ) + + case result do + :ok -> :ok + {:error, :existing} -> :ok + e -> raise e + end + end) + + :ok end end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 3972a03a9..cd569422b 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -24,5 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do defp add_scheme_opts(opts, _), do: opts - def after_request(_), do: :ok + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} + def get_conn(_uri, opts), do: {:ok, opts} end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex deleted file mode 100644 index ebacf7902..000000000 --- a/lib/pleroma/http/connection.ex +++ /dev/null @@ -1,124 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Connection do - @moduledoc """ - Configure Tesla.Client with default and customized adapter options. - """ - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - - require Logger - - @defaults [pool: :federation] - - @type ip_address :: ipv4_address() | ipv6_address() - @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} - @type ipv6_address :: - {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} - @type proxy_type() :: :socks4 | :socks5 - @type host() :: charlist() | ip_address() - - @doc """ - Merge default connection & adapter options with received ones. - """ - - @spec options(URI.t(), keyword()) :: keyword() - def options(%URI{} = uri, opts \\ []) do - @defaults - |> pool_timeout() - |> Keyword.merge(opts) - |> adapter_helper().options(uri) - end - - defp pool_timeout(opts) do - {config_key, default} = - if adapter() == Tesla.Adapter.Gun do - {:pools, Config.get([:pools, :default, :timeout])} - else - {:hackney_pools, 10_000} - end - - timeout = Config.get([config_key, opts[:pool], :timeout], default) - - Keyword.merge(opts, timeout: timeout) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter_helper().after_request(opts) - - defp adapter, do: Application.get_env(:tesla, :adapter) - - defp adapter_helper do - case adapter() do - Tesla.Adapter.Gun -> AdapterHelper.Gun - Tesla.Adapter.Hackney -> AdapterHelper.Hackney - _ -> AdapterHelper - end - end - - @spec parse_proxy(String.t() | tuple() | nil) :: - {:ok, host(), pos_integer()} - | {:ok, proxy_type(), host(), pos_integer()} - | {:error, atom()} - | nil - - def parse_proxy(nil), do: nil - - def parse_proxy(proxy) when is_binary(proxy) do - with [host, port] <- String.split(proxy, ":"), - {port, ""} <- Integer.parse(port) do - {:ok, parse_host(host), port} - else - {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - def parse_proxy(proxy) when is_tuple(proxy) do - with {type, host, port} <- proxy do - {:ok, type, parse_host(host), port} - else - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() - def parse_host(host) when is_list(host), do: host - def parse_host(host) when is_atom(host), do: to_charlist(host) - - def parse_host(host) when is_binary(host) do - host = to_charlist(host) - - case :inet.parse_address(host) do - {:error, :einval} -> host - {:ok, ip} -> ip - end - end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end -end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 66ca75367..b37b3fa89 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP do Wrapper for `Tesla.request/2`. """ - alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.AdapterHelper alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder alias Tesla.Client @@ -60,49 +60,30 @@ defmodule Pleroma.HTTP do {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - adapter_opts = Connection.options(uri, options[:adapter] || []) - options = put_in(options[:adapter], adapter_opts) - params = options[:params] || [] - request = build_request(method, headers, options, url, body, params) - - adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) - - pid = Process.whereis(adapter_opts[:pool]) - - pool_alive? = - if adapter == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end - - request_opts = - adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) - - response = request(client, request, request_opts) - - Connection.after_request(adapter_opts) - - response - end - - @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - - def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - - def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) - - def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout - ) + adapter_opts = AdapterHelper.options(uri, options[:adapter] || []) + + case AdapterHelper.get_conn(uri, adapter_opts) do + {:ok, adapter_opts} -> + options = put_in(options[:adapter], adapter_opts) + params = options[:params] || [] + request = build_request(method, headers, options, url, body, params) + + adapter = Application.get_env(:tesla, :adapter) + + client = Tesla.client(adapter_middlewares(adapter), adapter) + + maybe_limit( + fn -> + request(client, request) + end, + adapter, + adapter_opts + ) + + # Connection release is handled in a custom FollowRedirects middleware + err -> + err + end end @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} @@ -118,4 +99,19 @@ defmodule Pleroma.HTTP do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end + + @prefix Pleroma.Gun.ConnectionPool + defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do + ConcurrentLimiter.limit(:"#{@prefix}.#{opts[:pool] || :default}", fun) + end + + defp maybe_limit(fun, _, _) do + fun.() + end + + defp adapter_middlewares(Tesla.Adapter.Gun) do + [Pleroma.HTTP.Middleware.FollowRedirects] + end + + defp adapter_middlewares(_), do: [] end diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 7aacd9d80..31c9afe2a 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -413,6 +413,17 @@ defmodule Pleroma.ModerationLog do def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, + "action" => "approve", + "subject" => users + } + }) do + "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}" + end + + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, "nicknames" => nicknames, "tags" => tags, "action" => "tag" diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 32bcfcaba..0b171563b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -571,10 +571,7 @@ defmodule Pleroma.Notification do [ :self, :invisible, - :followers, - :follows, - :non_followers, - :non_follows, + :block_from_strangers, :recently_followed, :filtered ] @@ -595,45 +592,15 @@ defmodule Pleroma.Notification do end def skip?( - :followers, + :block_from_strangers, %Activity{} = activity, - %User{notification_settings: %{followers: false}} = user - ) do - actor = activity.data["actor"] - follower = User.get_cached_by_ap_id(actor) - User.following?(follower, user) - end - - def skip?( - :non_followers, - %Activity{} = activity, - %User{notification_settings: %{non_followers: false}} = user + %User{notification_settings: %{block_from_strangers: true}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?( - :follows, - %Activity{} = activity, - %User{notification_settings: %{follows: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - User.following?(user, followed) - end - - def skip?( - :non_follows, - %Activity{} = activity, - %User{notification_settings: %{non_follows: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - !User.following?(user, followed) - end - # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 3956bb727..3ff25118d 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -143,6 +143,10 @@ defmodule Pleroma.Object.Fetcher do {:error, "Object has been deleted"} -> nil + {:reject, reason} -> + Logger.info("Rejected #{id} while fetching: #{inspect(reason)}") + nil + e -> Logger.error("Error while fetching #{id}: #{inspect(e)}") nil diff --git a/lib/pleroma/plugs/frontend_static.ex b/lib/pleroma/plugs/frontend_static.ex new file mode 100644 index 000000000..f549ca75f --- /dev/null +++ b/lib/pleroma/plugs/frontend_static.ex @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.FrontendStatic do + require Pleroma.Constants + + @moduledoc """ + This is a shim to call `Plug.Static` but with runtime `from` configuration`. It dispatches to the different frontends. + """ + @behaviour Plug + + def file_path(path, frontend_type \\ :primary) do + if configuration = Pleroma.Config.get([:frontends, frontend_type]) do + instance_static_path = Pleroma.Config.get([:instance, :static_dir], "instance/static") + + Path.join([ + instance_static_path, + "frontends", + configuration["name"], + configuration["ref"], + path + ]) + else + nil + end + end + + def init(opts) do + opts + |> Keyword.put(:from, "__unconfigured_frontend_static_plug") + |> Plug.Static.init() + end + + def call(conn, opts) do + frontend_type = Map.get(opts, :frontend_type, :primary) + path = file_path("", frontend_type) + + if path do + conn + |> call_static(opts, path) + else + conn + end + end + + defp call_static(conn, opts, from) do + opts = + opts + |> Map.put(:from, from) + + Plug.Static.call(conn, opts) + end +end diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex index 7516f75c3..0fb57e422 100644 --- a/lib/pleroma/plugs/instance_static.ex +++ b/lib/pleroma/plugs/instance_static.ex @@ -16,28 +16,24 @@ defmodule Pleroma.Plugs.InstanceStatic do instance_path = Path.join(Pleroma.Config.get([:instance, :static_dir], "instance/static/"), path) - if File.exists?(instance_path) do - instance_path - else + frontend_path = Pleroma.Plugs.FrontendStatic.file_path(path, :primary) + + (File.exists?(instance_path) && instance_path) || + (frontend_path && File.exists?(frontend_path) && frontend_path) || Path.join(Application.app_dir(:pleroma, "priv/static/"), path) - end end def init(opts) do opts |> Keyword.put(:from, "__unconfigured_instance_static_plug") - |> Keyword.put(:at, "/__unconfigured_instance_static_plug") |> Plug.Static.init() end for only <- Pleroma.Constants.static_only_files() do - at = Plug.Router.Utils.split("/") - def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do call_static( conn, opts, - unquote(at), Pleroma.Config.get([:instance, :static_dir], "instance/static") ) end @@ -47,11 +43,10 @@ defmodule Pleroma.Plugs.InstanceStatic do conn end - defp call_static(conn, opts, at, from) do + defp call_static(conn, opts, from) do opts = opts |> Map.put(:from, from) - |> Map.put(:at, at) Plug.Static.call(conn, opts) end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex deleted file mode 100644 index acafe1bea..000000000 --- a/lib/pleroma/pool/connections.ex +++ /dev/null @@ -1,283 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Connections do - use GenServer - - alias Pleroma.Config - alias Pleroma.Gun - - require Logger - - @type domain :: String.t() - @type conn :: Pleroma.Gun.Conn.t() - - @type t :: %__MODULE__{ - conns: %{domain() => conn()}, - opts: keyword() - } - - defstruct conns: %{}, opts: [] - - @spec start_link({atom(), keyword()}) :: {:ok, pid()} - def start_link({name, opts}) do - GenServer.start_link(__MODULE__, opts, name: name) - end - - @impl true - def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} - - @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil - def checkin(url, name) - def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) - - def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :checkin_timeout], 250) - - GenServer.call(name, {:checkin, uri}, timeout) - end - - @spec alive?(atom()) :: boolean() - def alive?(name) do - if pid = Process.whereis(name) do - Process.alive?(pid) - else - false - end - end - - @spec get_state(atom()) :: t() - def get_state(name) do - GenServer.call(name, :state) - end - - @spec count(atom()) :: pos_integer() - def count(name) do - GenServer.call(name, :count) - end - - @spec get_unused_conns(atom()) :: [{domain(), conn()}] - def get_unused_conns(name) do - GenServer.call(name, :unused_conns) - end - - @spec checkout(pid(), pid(), atom()) :: :ok - def checkout(conn, pid, name) do - GenServer.cast(name, {:checkout, conn, pid}) - end - - @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok - def add_conn(name, key, conn) do - GenServer.cast(name, {:add_conn, key, conn}) - end - - @spec remove_conn(atom(), String.t()) :: :ok - def remove_conn(name, key) do - GenServer.cast(name, {:remove_conn, key}) - end - - @impl true - def handle_cast({:add_conn, key, conn}, state) do - state = put_in(state.conns[key], conn) - - Process.monitor(conn.conn) - {:noreply, state} - end - - @impl true - def handle_cast({:checkout, conn_pid, pid}, state) do - state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid), - used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = if used_by == [], do: :idle, else: conn.conn_state - - put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) - else - false -> - Logger.debug("checkout for closed conn #{inspect(conn_pid)}") - state - - nil -> - Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") - state - end - - {:noreply, state} - end - - @impl true - def handle_cast({:remove_conn, key}, state) do - state = put_in(state.conns, Map.delete(state.conns, key)) - {:noreply, state} - end - - @impl true - def handle_call({:checkin, uri}, from, state) do - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - case state.conns[key] do - %{conn: pid, gun_state: :up} = conn -> - time = :os.system_time(:second) - last_reference = time - conn.last_reference - crf = crf(last_reference, 100, conn.crf) - - state = - put_in(state.conns[key], %{ - conn - | last_reference: time, - crf: crf, - conn_state: :active, - used_by: [from | conn.used_by] - }) - - {:reply, pid, state} - - %{gun_state: :down} -> - {:reply, nil, state} - - nil -> - {:reply, nil, state} - end - end - - @impl true - def handle_call(:state, _from, state), do: {:reply, state, state} - - @impl true - def handle_call(:count, _from, state) do - {:reply, Enum.count(state.conns), state} - end - - @impl true - def handle_call(:unused_conns, _from, state) do - unused_conns = - state.conns - |> Enum.filter(&filter_conns/1) - |> Enum.sort(&sort_conns/2) - - {:reply, unused_conns, state} - end - - defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true - defp filter_conns(_), do: false - - defp sort_conns({_, c1}, {_, c2}) do - c1.crf <= c2.crf and c1.last_reference <= c2.last_reference - end - - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - - state = - with {key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do - put_in(state.conns[key], %{ - conn - | gun_state: :up, - conn_state: :active, - retries: 0 - }) - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - :ok = Gun.close(conn_pid) - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 1) - # we can't get info on this pid, because pid is dead - state = - with {key, conn} <- find_conn(state.conns, conn_pid), - {true, key} <- {Process.alive?(conn_pid), key} do - if conn.retries == retries do - :ok = Gun.close(conn.conn) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - put_in(state.conns[key], %{ - conn - | gun_state: :down, - retries: conn.retries + 1 - }) - end - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - Logger.debug(":gun_down for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do - Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") - - state = - with {key, conn} <- find_conn(state.conns, conn_pid) do - Enum.each(conn.used_by, fn {pid, _ref} -> - Process.exit(pid, reason) - end) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - nil -> - Logger.debug(":DOWN for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - defp find_conn(conns, conn_pid) do - Enum.find(conns, fn {_key, conn} -> - conn.conn == conn_pid - end) - end - - defp find_conn(conns, conn_pid, conn_key) do - Enum.find(conns, fn {key, conn} -> - key == conn_key and conn.conn == conn_pid - end) - end - - def crf(current, steps, crf) do - 1 + :math.pow(0.5, current / steps) * crf - end -end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex deleted file mode 100644 index 21a6fbbc5..000000000 --- a/lib/pleroma/pool/pool.ex +++ /dev/null @@ -1,22 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool do - def child_spec(opts) do - poolboy_opts = - opts - |> Keyword.put(:worker_module, Pleroma.Pool.Request) - |> Keyword.put(:name, {:local, opts[:name]}) - |> Keyword.put(:size, opts[:size]) - |> Keyword.put(:max_overflow, opts[:max_overflow]) - - %{ - id: opts[:id] || {__MODULE__, make_ref()}, - start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, - restart: :permanent, - shutdown: 5000, - type: :worker - } - end -end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex deleted file mode 100644 index 3fb930db7..000000000 --- a/lib/pleroma/pool/request.ex +++ /dev/null @@ -1,65 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Request do - use GenServer - - require Logger - - def start_link(args) do - GenServer.start_link(__MODULE__, args) - end - - @impl true - def init(_), do: {:ok, []} - - @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: - {:ok, Tesla.Env.t()} | {:error, any()} - def execute(pid, client, request, timeout) do - GenServer.call(pid, {:execute, client, request}, timeout) - end - - @impl true - def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request(client, request) - - {:reply, response, state} - end - - @impl true - def handle_info({:gun_data, _conn, _stream, _, _}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_up, _conn, _protocol}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_error, _conn, _stream, _error}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info(msg, state) do - Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") - {:noreply, state} - end -end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex deleted file mode 100644 index faf646cb2..000000000 --- a/lib/pleroma/pool/supervisor.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Supervisor do - use Supervisor - - alias Pleroma.Config - alias Pleroma.Pool - - def start_link(args) do - Supervisor.start_link(__MODULE__, args, name: __MODULE__) - end - - def init(_) do - conns_child = %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - - Supervisor.init([conns_child | pools()], strategy: :one_for_one) - end - - defp pools do - pools = Config.get(:pools) - - pools = - if Config.get([Pleroma.Upload, :proxy_remote]) == false do - Keyword.delete(pools, :upload) - else - pools - end - - for {pool_name, pool_opts} <- pools do - pool_opts - |> Keyword.put(:id, {Pool, pool_name}) - |> Keyword.put(:name, pool_name) - |> Pool.child_spec() - end - end -end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index e81ea8bde..d5a339681 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -5,6 +5,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client + alias Pleroma.Gun.ConnectionPool + @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() @@ -31,6 +33,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do if is_map(response.body) and method != :head do {:ok, response.status, response.headers, response.body} else + conn_pid = response.opts[:adapter][:conn] + ConnectionPool.release_conn(conn_pid) {:ok, response.status, response.headers} end else @@ -41,15 +45,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() - def stream_body(%{pid: pid, opts: opts, fin: true}) do - # if connection was reused, but in tesla were redirects, - # tesla returns new opened connection, which must be closed manually - if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) - # if there were redirects we need to checkout old conn - conn = opts[:old_conn] || opts[:conn] - - if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) - + def stream_body(%{pid: pid, fin: true}) do + ConnectionPool.release_conn(pid) :done end @@ -74,8 +71,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec close(map) :: :ok | no_return() def close(%{pid: pid}) do - adapter = check_adapter() - adapter.close(pid) + ConnectionPool.release_conn(pid) end defp check_adapter do diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 28ad4c846..0de4e2309 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -165,6 +165,9 @@ defmodule Pleroma.ReverseProxy do {:ok, code, _, _} -> {:error, {:invalid_http_response, code}} + {:ok, code, _} -> + {:error, {:invalid_http_response, code}} + {:error, error} -> {:error, error} end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex new file mode 100644 index 000000000..4cacae02f --- /dev/null +++ b/lib/pleroma/telemetry/logger.ex @@ -0,0 +1,76 @@ +defmodule Pleroma.Telemetry.Logger do + @moduledoc "Transforms Pleroma telemetry events to logs" + + require Logger + + @events [ + [:pleroma, :connection_pool, :reclaim, :start], + [:pleroma, :connection_pool, :reclaim, :stop], + [:pleroma, :connection_pool, :provision_failure], + [:pleroma, :connection_pool, :client_death] + ] + def attach do + :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) + end + + # Passing anonymous functions instead of strings to logger is intentional, + # that way strings won't be concatenated if the message is going to be thrown + # out anyway due to higher log level configured + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :start], + _, + %{max_connections: max_connections, reclaim_max: reclaim_max}, + _ + ) do + Logger.debug(fn -> + "Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{ + reclaim_max + } connections" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: reclaimed_count}, + _, + _ + ) do + Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end) + end + + def handle_event( + [:pleroma, :connection_pool, :provision_failure], + %{opts: [key | _]}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :client_death], + %{client_pid: client_pid, reason: reason}, + %{key: key}, + _ + ) do + Logger.warn(fn -> + "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{ + inspect(reason) + }" + end) + end +end diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex new file mode 100644 index 000000000..5a7032215 --- /dev/null +++ b/lib/pleroma/tesla/middleware/follow_redirects.ex @@ -0,0 +1,110 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex> +# Copyright © 2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Middleware.FollowRedirects do + @moduledoc """ + Pool-aware version of https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex + + Follow 3xx redirects + ## Options + - `:max_redirects` - limit number of redirects (default: `5`) + """ + + alias Pleroma.Gun.ConnectionPool + + @behaviour Tesla.Middleware + + @max_redirects 5 + @redirect_statuses [301, 302, 303, 307, 308] + + @impl Tesla.Middleware + def call(env, next, opts \\ []) do + max = Keyword.get(opts, :max_redirects, @max_redirects) + + redirect(env, next, max) + end + + defp redirect(env, next, left) do + opts = env.opts[:adapter] + + case Tesla.run(env, next) do + {:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 -> + release_conn(opts) + + case Tesla.get_header(res, "location") do + nil -> + {:ok, res} + + location -> + location = parse_location(location, res) + + case get_conn(location, opts) do + {:ok, opts} -> + %{env | opts: Keyword.put(env.opts, :adapter, opts)} + |> new_request(res.status, location) + |> redirect(next, left - 1) + + e -> + e + end + end + + {:ok, %{status: status}} when status in @redirect_statuses -> + release_conn(opts) + {:error, {__MODULE__, :too_many_redirects}} + + {:error, _} = e -> + release_conn(opts) + e + + other -> + unless opts[:body_as] == :chunks do + release_conn(opts) + end + + other + end + end + + defp get_conn(location, opts) do + uri = URI.parse(location) + + case ConnectionPool.get_conn(uri, opts) do + {:ok, conn} -> + {:ok, Keyword.merge(opts, conn: conn)} + + e -> + e + end + end + + defp release_conn(opts) do + ConnectionPool.release_conn(opts[:conn]) + end + + # The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally + # requested resource is not available, however a related resource (or another redirect) + # available via GET is available at the specified location. + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []} + + # The 307 (Temporary Redirect) status code indicates that the target + # resource resides temporarily under a different URI and the user agent + # MUST NOT change the request method (...) + # https://tools.ietf.org/html/rfc7231#section-6.4.7 + defp new_request(env, 307, location), do: %{env | url: location} + + defp new_request(env, _, location), do: %{env | url: location, query: []} + + defp parse_location("https://" <> _rest = location, _env), do: location + defp parse_location("http://" <> _rest = location, _env), do: location + + defp parse_location(location, env) do + env.url + |> URI.parse() + |> URI.merge(location) + |> URI.to_string() + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9240e912d..dcf6ebee2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -42,7 +42,12 @@ defmodule Pleroma.User do require Logger @type t :: %__MODULE__{} - @type account_status :: :active | :deactivated | :password_reset_pending | :confirmation_pending + @type account_status :: + :active + | :deactivated + | :password_reset_pending + | :confirmation_pending + | :approval_pending @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength @@ -106,6 +111,8 @@ defmodule Pleroma.User do field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:password_reset_pending, :boolean, default: false) + field(:approval_pending, :boolean, default: false) + field(:registration_reason, :string, default: nil) field(:confirmation_token, :string, default: nil) field(:default_scope, :string, default: "public") field(:domain_blocks, {:array, :string}, default: []) @@ -262,6 +269,7 @@ defmodule Pleroma.User do @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated def account_status(%User{password_reset_pending: true}), do: :password_reset_pending + def account_status(%User{approval_pending: true}), do: :approval_pending def account_status(%User{confirmation_pending: true}) do if Config.get([:instance, :account_activation_required]) do @@ -633,6 +641,7 @@ defmodule Pleroma.User do def register_changeset(struct, params \\ %{}, opts \\ []) do bio_limit = Config.get([:instance, :user_bio_length], 5000) name_limit = Config.get([:instance, :user_name_length], 100) + reason_limit = Config.get([:instance, :registration_reason_length], 500) params = Map.put_new(params, :accepts_chat_messages, true) need_confirmation? = @@ -642,8 +651,16 @@ defmodule Pleroma.User do opts[:need_confirmation] end + need_approval? = + if is_nil(opts[:need_approval]) do + Config.get([:instance, :account_approval_required]) + else + opts[:need_approval] + end + struct |> confirmation_changeset(need_confirmation: need_confirmation?) + |> approval_changeset(need_approval: need_approval?) |> cast(params, [ :bio, :raw_bio, @@ -653,7 +670,8 @@ defmodule Pleroma.User do :password, :password_confirmation, :emoji, - :accepts_chat_messages + :accepts_chat_messages, + :registration_reason ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) @@ -664,6 +682,7 @@ defmodule Pleroma.User do |> validate_format(:email, @email_regex) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) + |> validate_length(:registration_reason, max: reason_limit) |> maybe_validate_required_email(opts[:external]) |> put_password_hash |> put_ap_id() @@ -713,27 +732,52 @@ defmodule Pleroma.User do def post_register_action(%User{} = user) do with {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), - {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user), + {:ok, _} <- send_welcome_email(user), + {:ok, _} <- send_welcome_message(user), {:ok, _} <- try_send_confirmation_email(user) do {:ok, user} end end - def try_send_confirmation_email(%User{} = user) do - if user.confirmation_pending && - Config.get([:instance, :account_activation_required]) do - user - |> Pleroma.Emails.UserEmail.account_confirmation_email() - |> Pleroma.Emails.Mailer.deliver_async() + def send_welcome_message(user) do + if User.WelcomeMessage.enabled?() do + User.WelcomeMessage.post_message(user) + {:ok, :enqueued} + else + {:ok, :noop} + end + end + + def send_welcome_email(%User{email: email} = user) when is_binary(email) do + if User.WelcomeEmail.enabled?() do + User.WelcomeEmail.send_email(user) + {:ok, :enqueued} + else + {:ok, :noop} + end + end + + def send_welcome_email(_), do: {:ok, :noop} + @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} + def try_send_confirmation_email(%User{confirmation_pending: true} = user) do + if Config.get([:instance, :account_activation_required]) do + send_confirmation_email(user) {:ok, :enqueued} else {:ok, :noop} end end - def try_send_confirmation_email(users) do - Enum.each(users, &try_send_confirmation_email/1) + def try_send_confirmation_email(_), do: {:ok, :noop} + + @spec send_confirmation_email(Uset.t()) :: User.t() + def send_confirmation_email(%User{} = user) do + user + |> Pleroma.Emails.UserEmail.account_confirmation_email() + |> Pleroma.Emails.Mailer.deliver_async() + + user end def needs_update?(%User{local: true}), do: false @@ -1469,6 +1513,19 @@ defmodule Pleroma.User do end end + def approve(users) when is_list(users) do + Repo.transaction(fn -> + Enum.map(users, fn user -> + with {:ok, user} <- approve(user), do: user + end) + end) + end + + def approve(%User{} = user) do + change(user, approval_pending: false) + |> update_and_set_cache() + end + def update_notification_settings(%User{} = user, settings) do user |> cast(%{notification_settings: settings}, []) @@ -1495,12 +1552,17 @@ defmodule Pleroma.User do defp delete_or_deactivate(%User{local: true} = user) do status = account_status(user) - if status == :confirmation_pending do - delete_and_invalidate_cache(user) - else - user - |> change(%{deactivated: true, email: nil}) - |> update_and_set_cache() + case status do + :confirmation_pending -> + delete_and_invalidate_cache(user) + + :approval_pending -> + delete_and_invalidate_cache(user) + + _ -> + user + |> change(%{deactivated: true, email: nil}) + |> update_and_set_cache() end end @@ -2153,6 +2215,12 @@ defmodule Pleroma.User do cast(user, params, [:confirmation_pending, :confirmation_token]) end + @spec approval_changeset(User.t(), keyword()) :: Changeset.t() + def approval_changeset(user, need_approval: need_approval?) do + params = if need_approval?, do: %{approval_pending: true}, else: %{approval_pending: false} + cast(user, params, [:approval_pending]) + end + def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do if id not in user.pinned_activities do max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex index 4bd55e139..7d9e8a000 100644 --- a/lib/pleroma/user/notification_setting.ex +++ b/lib/pleroma/user/notification_setting.ex @@ -10,21 +10,15 @@ defmodule Pleroma.User.NotificationSetting do @primary_key false embedded_schema do - field(:followers, :boolean, default: true) - field(:follows, :boolean, default: true) - field(:non_follows, :boolean, default: true) - field(:non_followers, :boolean, default: true) - field(:privacy_option, :boolean, default: false) + field(:block_from_strangers, :boolean, default: false) + field(:hide_notification_contents, :boolean, default: false) end def changeset(schema, params) do schema |> cast(prepare_attrs(params), [ - :followers, - :follows, - :non_follows, - :non_followers, - :privacy_option + :block_from_strangers, + :hide_notification_contents ]) end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 66ffe9090..45553cb6c 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -42,6 +42,7 @@ defmodule Pleroma.User.Query do external: boolean(), active: boolean(), deactivated: boolean(), + need_approval: boolean(), is_admin: boolean(), is_moderator: boolean(), super_users: boolean(), @@ -146,6 +147,10 @@ defmodule Pleroma.User.Query do |> where([u], not is_nil(u.nickname)) end + defp compose_query({:need_approval, _}, query) do + where(query, [u], u.approval_pending) + end + defp compose_query({:followers, %User{id: id}}, query) do query |> where([u], u.id != ^id) diff --git a/lib/pleroma/user/welcome_email.ex b/lib/pleroma/user/welcome_email.ex new file mode 100644 index 000000000..5322000d4 --- /dev/null +++ b/lib/pleroma/user/welcome_email.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.WelcomeEmail do + @moduledoc """ + The module represents the functions to send welcome email. + """ + + alias Pleroma.Config + alias Pleroma.Emails + alias Pleroma.User + + import Pleroma.Config.Helpers, only: [instance_name: 0] + + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:welcome, :email, :enabled], false) + + @spec send_email(User.t()) :: {:ok, Oban.Job.t()} + def send_email(%User{} = user) do + user + |> Emails.UserEmail.welcome(email_options(user)) + |> Emails.Mailer.deliver_async() + end + + defp email_options(user) do + bindings = [user: user, instance_name: instance_name()] + + %{} + |> add_sender(Config.get([:welcome, :email, :sender], nil)) + |> add_option(:subject, bindings) + |> add_option(:html, bindings) + |> add_option(:text, bindings) + end + + defp add_option(opts, option, bindings) do + [:welcome, :email, option] + |> Config.get(nil) + |> eval_string(bindings) + |> merge_options(opts, option) + end + + defp add_sender(opts, {_name, _email} = sender) do + merge_options(sender, opts, :sender) + end + + defp add_sender(opts, sender) when is_binary(sender) do + add_sender(opts, {instance_name(), sender}) + end + + defp add_sender(opts, _), do: opts + + defp merge_options(nil, options, _option), do: options + + defp merge_options(value, options, option) do + Map.merge(options, %{option => value}) + end + + defp eval_string(nil, _), do: nil + defp eval_string("", _), do: nil + defp eval_string(str, bindings), do: EEx.eval_string(str, bindings) +end diff --git a/lib/pleroma/user/welcome_message.ex b/lib/pleroma/user/welcome_message.ex index f8f520285..86e1c0678 100644 --- a/lib/pleroma/user/welcome_message.ex +++ b/lib/pleroma/user/welcome_message.ex @@ -3,32 +3,45 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.WelcomeMessage do + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI - def post_welcome_message_to_user(user) do - with %User{} = sender_user <- welcome_user(), - message when is_binary(message) <- welcome_message() do - CommonAPI.post(sender_user, %{ + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:welcome, :direct_message, :enabled], false) + + @spec post_message(User.t()) :: {:ok, Pleroma.Activity.t() | nil} + def post_message(user) do + [:welcome, :direct_message, :sender_nickname] + |> Config.get(nil) + |> fetch_sender() + |> do_post(user, welcome_message()) + end + + defp do_post(%User{} = sender, %User{nickname: nickname}, message) + when is_binary(message) do + CommonAPI.post( + sender, + %{ visibility: "direct", - status: "@#{user.nickname}\n#{message}" - }) - else - _ -> {:ok, nil} - end + status: "@#{nickname}\n#{message}" + } + ) end - defp welcome_user do - with nickname when is_binary(nickname) <- - Pleroma.Config.get([:instance, :welcome_user_nickname]), - %User{local: true} = user <- User.get_cached_by_nickname(nickname) do + defp do_post(_sender, _recipient, _message), do: {:ok, nil} + + defp fetch_sender(nickname) when is_binary(nickname) do + with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do user else _ -> nil end end + defp fetch_sender(_), do: nil + defp welcome_message do - Pleroma.Config.get([:instance, :welcome_message]) + Config.get([:welcome, :direct_message, :message], nil) end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9d13a06c4..fe62673dc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1359,6 +1359,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} + {:error, {:reject, reason} = e} -> + Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") + {:error, e} + {:error, e} -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 0270b96ae..b96388489 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do if score < 0.8 do {:ok, message} else - {:reject, nil} + {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index a7e187b5e..b22464111 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -39,14 +39,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do {:ok, message} {:old_user, false} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"} {:error, _} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Failed to get or fetch user by ap_id"} e -> - Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}") - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Unhandled error #{inspect(e)}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 2627a0007..3bf70b894 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -27,7 +27,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => "Create", "object" => child_object} = object) do + def filter(%{"type" => "Create", "object" => child_object} = object) + when is_map(child_object) do child = child_object["inReplyTo"] |> Object.normalize(child_object["inReplyTo"]) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index f6b2c4415..9ba07b4e3 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do defp reject_message(message, threshold) when threshold > 0 do with {_, recipients} <- get_recipient_count(message) do if recipients > threshold do - {:reject, nil} + {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"} else {:ok, message} end @@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do {:ok, message} <- delist_message(message, delist_threshold) do {:ok, message} else - _e -> {:reject, nil} + e -> e end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 88b0d2b39..15e09dcf0 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> string_matches?(content, pattern) or string_matches?(summary, pattern) end) do - {:reject, nil} + {:reject, "[KeywordPolicy] Matches with rejected keyword"} else {:ok, message} end @@ -89,8 +89,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do {:ok, message} <- check_replace(message) do {:ok, message} else - _e -> - {:reject, nil} + {:reject, nil} -> {:reject, "[KeywordPolicy] "} + {:reject, _} = e -> e + _e -> {:reject, "[KeywordPolicy] "} end end diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 06f003921..7910ca131 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -12,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) recipients = (message["to"] || []) ++ (message["cc"] || []) - if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do - {:reject, nil} + if rejected_mention = + Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do + {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"} else {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index a62914135..5f111c72f 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do defp check_reject(message, actions) do if :reject in actions do - {:reject, nil} + {:reject, "[ObjectAgePolicy]"} else {:ok, message} end @@ -47,9 +47,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} @@ -69,9 +68,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 4fd63106d..0b9ed2224 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do {:ok, object} true -> - {:reject, nil} + {:reject, "[RejectNonPublic] visibility: #{visibility}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 70a2ca053..b77b8c7b4 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do accepts == [] -> {:ok, object} actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} - true -> {:reject, nil} + true -> {:reject, "[SimplePolicy] host not in accept list"} end end @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(rejects, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject list"} else {:ok, object} end @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(report_removal, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in report_removal list"} else {:ok, object} end @@ -159,7 +159,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(reject_deletes, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject_deletes list"} else {:ok, object} end @@ -177,7 +177,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end @@ -191,7 +193,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_banner_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c310462cb..febabda08 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -134,12 +134,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do if user.local == true do {:ok, message} else - {:reject, nil} + {:reject, + "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"} end end - defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), - do: {:reject, nil} + defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}), + do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"} defp process_tag(_, message), do: {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 651aed70f..1a28f2ba2 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do if actor in allow_list do {:ok, object} else - {:reject, nil} + {:reject, "[UserAllowListPolicy] #{actor} not in the list"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 6167a74e2..a6c545570 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -11,22 +11,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do with {:ok, _} <- filter(child_message) do {:ok, message} else - {:reject, nil} -> - {:reject, nil} + {:reject, _} = e -> e end end def filter(%{"type" => message_type} = message) do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), - true <- - Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type), - false <- - length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), + {_, true} <- + {:accepted, + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)}, + {_, false} <- + {:rejected, + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)}, {:ok, _} <- filter(message["object"]) do {:ok, message} else - _ -> {:reject, nil} + {:reject, _} = e -> e + {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} + {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} + _ -> {:reject, "[VocabularyPolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index a24aaf00c..e1114a44d 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do the system. """ + alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object alias Pleroma.User @@ -74,6 +75,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do |> UndoValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) + undone_object = Activity.get_by_ap_id(object["object"]) + + meta = + meta + |> Keyword.put(:object_data, undone_object.data) + {:ok, object, meta} end end diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 6875c47f6..36e325c37 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -52,6 +52,13 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do do_not_federate = meta[:do_not_federate] || !Config.get([:instance, :federating]) if !do_not_federate && local do + activity = + if object = Keyword.get(meta, :object_data) do + %{activity | data: Map.put(activity.data, "object", object)} + else + activity + end + Federator.publish(activity) {:ok, :federated} else diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index b70cbd043..d88f7f3ee 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -49,7 +49,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do Logger.debug("Federating #{id} to #{inbox}") - %{host: host, path: path} = URI.parse(inbox) + + uri = URI.parse(inbox) digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) @@ -57,8 +58,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do signature = Pleroma.Signature.sign(actor, %{ - "(request-target)": "post #{path}", - host: host, + "(request-target)": "post #{uri.path}", + host: signature_host(uri), "content-length": byte_size(json), digest: digest, date: date @@ -76,8 +77,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", digest} ] ) do - if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], - do: Instances.set_reachable(inbox) + if not Map.has_key?(params, :unreachable_since) || params[:unreachable_since] do + Instances.set_reachable(inbox) + end result else @@ -96,6 +98,14 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> publish_one() end + defp signature_host(%URI{port: port, scheme: scheme, host: host}) do + if port == URI.default_port(scheme) do + host + else + "#{host}:#{port}" + end + end + defp should_federate?(inbox, public) do if public do true diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index edabe1130..f85a26679 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -183,7 +183,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation"]) else e -> - Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index dfae602df..713b0ca1f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -719,15 +719,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do case Activity.get_by_ap_id_with_object(id) do %Activity{} = activity -> + activity_actor = User.get_by_ap_id(activity.object.data["actor"]) + %{ "type" => "Note", "id" => activity.data["id"], "content" => activity.object.data["content"], "published" => activity.object.data["published"], "actor" => - AccountView.render("show.json", %{ - user: User.get_by_ap_id(activity.object.data["actor"]) - }) + AccountView.render( + "show.json", + %{user: activity_actor, skip_visibility_check: true} + ) } _ -> diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index e5f14269a..aa2af1ab5 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :user_toggle_activation, :user_activate, :user_deactivate, + :user_approve, :tag_users, :untag_users, :right_add, @@ -303,6 +304,21 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{users: Keyword.values(updated_users)}) end + def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.approve(users) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "approve" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: updated_users}) + end + def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do with {:ok, _} <- User.tag(nicknames, tags) do ModerationLog.insert_log(%{ @@ -345,12 +361,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do json( conn, - AccountView.render("index.json", users: users, count: count, page_size: page_size) + AccountView.render("index.json", + users: users, + count: count, + page_size: page_size + ) ) end end - @filters ~w(local external active deactivated is_admin is_moderator) + @filters ~w(local external active deactivated need_approval is_admin is_moderator) @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} @@ -616,29 +636,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) User.toggle_confirmation(users) - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "confirm_email" - }) + ModerationLog.insert_log(%{actor: admin, subject: users, action: "confirm_email"}) json(conn, "") end def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) - - User.try_send_confirmation_email(users) + users = + Enum.map(nicknames, fn nickname -> + nickname + |> User.get_cached_by_nickname() + |> User.send_confirmation_email() + end) - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "resend_confirmation_email" - }) + ModerationLog.insert_log(%{actor: admin, subject: users, action: "resend_confirmation_email"}) json(conn, "") end diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index e1e929632..333e72e42 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -77,7 +77,9 @@ defmodule Pleroma.Web.AdminAPI.AccountView do "roles" => User.roles(user), "tags" => user.tags || [], "confirmation_pending" => user.confirmation_pending, - "url" => user.uri || user.ap_id + "approval_pending" => user.approval_pending, + "url" => user.uri || user.ap_id, + "registration_reason" => user.registration_reason } end @@ -105,7 +107,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do end def merge_account_views(%User{} = user) do - MastodonAPI.AccountView.render("show.json", %{user: user}) + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user})) end diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 952d9347b..50c8e0242 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -159,6 +159,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do "Accounts which follow the given account, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{ @@ -177,6 +178,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do "Accounts which the given account is following, if network is not hidden by the account owner.", parameters: [ %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, + Operation.parameter(:id, :query, :string, "ID of the resource owner"), with_relationships_param() | pagination_params() ], responses: %{200 => Operation.response("Accounts", "application/json", array_of_accounts())} diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex index cf299bfc2..b1a0d26ab 100644 --- a/lib/pleroma/web/api_spec/operations/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex @@ -300,11 +300,11 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "content" => "Check this out :firefox:", "id" => "13", "chat_id" => "1", - "actor_id" => "someflakeid", + "account_id" => "someflakeid", "unread" => false }, %{ - "actor_id" => "someflakeid", + "account_id" => "someflakeid", "content" => "Whats' up?", "id" => "12", "chat_id" => "1", diff --git a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex index 049bcf931..1e0da8209 100644 --- a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex +++ b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do } end + # Supporting domain query parameter is deprecated in Mastodon API def create_operation do %Operation{ tags: ["domain_blocks"], @@ -45,11 +46,13 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do """, operationId: "DomainBlockController.create", requestBody: domain_block_request(), + parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")], security: [%{"oAuth" => ["follow", "write:blocks"]}], responses: %{200 => empty_object_response()} } end + # Supporting domain query parameter is deprecated in Mastodon API def delete_operation do %Operation{ tags: ["domain_blocks"], @@ -57,6 +60,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do description: "Remove a domain block, if it exists in the user's array of blocked domains.", operationId: "DomainBlockController.delete", requestBody: domain_block_request(), + parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")], security: [%{"oAuth" => ["follow", "write:blocks"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -71,10 +75,9 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do type: :object, properties: %{ domain: %Schema{type: :string} - }, - required: [:domain] + } }, - required: true, + required: false, example: %{ "domain" => "facebook.com" } diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index cf148bc9d..ca79f0747 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -90,11 +90,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do notification_settings: %Schema{ type: :object, properties: %{ - followers: %Schema{type: :boolean}, - follows: %Schema{type: :boolean}, - non_followers: %Schema{type: :boolean}, - non_follows: %Schema{type: :boolean}, - privacy_option: %Schema{type: :boolean} + block_from_strangers: %Schema{type: :boolean}, + hide_notification_contents: %Schema{type: :boolean} } }, relationship: AccountRelationship, @@ -182,11 +179,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "unread_conversation_count" => 0, "tags" => [], "notification_settings" => %{ - "followers" => true, - "follows" => true, - "non_followers" => true, - "non_follows" => true, - "privacy_option" => false + "block_from_strangers" => false, + "hide_notification_contents" => false }, "relationship" => %{ "blocked_by" => false, diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex index bce27897f..3b1469c19 100644 --- a/lib/pleroma/web/chat_channel.ex +++ b/lib/pleroma/web/chat_channel.ex @@ -4,8 +4,10 @@ defmodule Pleroma.Web.ChatChannel do use Phoenix.Channel + alias Pleroma.User alias Pleroma.Web.ChatChannel.ChatChannelState + alias Pleroma.Web.MastodonAPI.AccountView def join("chat:public", _message, socket) do send(self(), :after_join) @@ -22,9 +24,9 @@ defmodule Pleroma.Web.ChatChannel do if String.length(text) in 1..Pleroma.Config.get([:instance, :chat_limit]) do author = User.get_cached_by_nickname(user_name) - author = Pleroma.Web.MastodonAPI.AccountView.render("show.json", user: author) + author_json = AccountView.render("show.json", user: author, skip_visibility_check: true) - message = ChatChannelState.add_message(%{text: text, author: author}) + message = ChatChannelState.add_message(%{text: text, author: author_json}) broadcast!(socket, "new_msg", message) end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 226d42c2c..527fb288d 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -28,6 +28,17 @@ defmodule Pleroma.Web.Endpoint do } ) + # Careful! No `only` restriction here, as we don't know what frontends contain. + plug(Pleroma.Plugs.FrontendStatic, + at: "/", + frontend_type: :primary, + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } + ) + # Serve at "/" the static files from "priv/static" directory. # # You should set gzip to true if you are running phoenix.digest diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index d56f43818..9cd334a33 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Web.Feed.UserController do "atom" end - with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do + with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do activities = %{ type: ["Create"], @@ -71,6 +71,7 @@ defmodule Pleroma.Web.Feed.UserController do render_error(conn, :not_found, "Not found") end + def errors(conn, {:fetch_user, %User{local: false}}), do: errors(conn, {:error, :not_found}) def errors(conn, {:fetch_user, nil}), do: errors(conn, {:error, :not_found}) def errors(conn, _) do diff --git a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex index 825b231ab..9c2d093cd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex @@ -32,9 +32,19 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do json(conn, %{}) end + def create(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do + User.block_domain(blocker, domain) + json(conn, %{}) + end + @doc "DELETE /api/v1/domain_blocks" def delete(%{assigns: %{user: blocker}, body_params: %{domain: domain}} = conn, _params) do User.unblock_domain(blocker, domain) json(conn, %{}) end + + def delete(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do + User.unblock_domain(blocker, domain) + json(conn, %{}) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 29affa7d5..5a983db39 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -93,7 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do AccountView.render("index.json", users: accounts, for: options[:for_user], - as: :user, embed_relationships: options[:embed_relationships] ) end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 12be530c9..9bb2ef117 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -172,6 +172,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do with_direct_conversation_id: true ) else + {:error, {:reject, message}} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{error: message}) + {:error, message} -> conn |> put_status(:unprocessable_entity) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index bc9745044..864c0417f 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -27,21 +27,40 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do UserRelationship.view_relationships_option(reading_user, users) end - opts = Map.put(opts, :relationships, relationships_opt) + opts = + opts + |> Map.merge(%{relationships: relationships_opt, as: :user}) + |> Map.delete(:users) users |> render_many(AccountView, "show.json", opts) |> Enum.filter(&Enum.any?/1) end - def render("show.json", %{user: user} = opts) do - if User.visible_for(user, opts[:for]) == :visible do + @doc """ + Renders specified user account. + :skip_visibility_check option skips visibility check and renders any user (local or remote) + regardless of [:pleroma, :restrict_unauthenticated] setting. + :for option specifies the requester and can be a User record or nil. + Only use `user: user, for: user` when `user` is the actual requester of own profile. + """ + def render("show.json", %{user: _user, skip_visibility_check: true} = opts) do + do_render("show.json", opts) + end + + def render("show.json", %{user: user, for: for_user_or_nil} = opts) do + if User.visible_for(user, for_user_or_nil) == :visible do do_render("show.json", opts) else %{} end end + def render("show.json", _) do + raise "In order to prevent account accessibility issues, " <> + ":skip_visibility_check or :for option is required." + end + def render("mention.json", %{user: user}) do %{ id: to_string(user.id), diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 06f0c1728..a91994915 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do %{ id: participation.id |> to_string(), - accounts: render(AccountView, "index.json", users: users, as: :user), + accounts: render(AccountView, "index.json", users: users, for: user), unread: !participation.read, last_status: render(StatusView, "show.json", diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 5deb0d7ed..ea2d3aa9c 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do thumbnail: Keyword.get(instance, :instance_thumbnail), languages: ["en"], registrations: Keyword.get(instance, :registrations_open), + approval_required: Keyword.get(instance, :account_approval_required), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), poll_limits: Keyword.get(instance, :poll_limits), @@ -41,7 +42,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do account_activation_required: Keyword.get(instance, :account_activation_required), features: features(), federation: federation(), - fields_limits: fields_limits() + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]) }, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index fa9d695f3..91b41ef59 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -297,13 +297,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do emoji_reactions = with %{data: %{"reactions" => emoji_reactions}} <- object do - Enum.map(emoji_reactions, fn [emoji, users] -> - %{ - name: emoji, - count: length(users), - me: !!(opts[:for] && opts[:for].ap_id in users) - } + Enum.map(emoji_reactions, fn + [emoji, users] when is_list(users) -> + build_emoji_map(emoji, users, opts[:for]) + + {emoji, users} when is_list(users) -> + build_emoji_map(emoji, users, opts[:for]) + + _ -> + nil end) + |> Enum.reject(&is_nil/1) else _ -> [] end @@ -545,4 +549,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}), do: id in pinned_activities + + defp build_emoji_map(emoji, users, current_user) do + %{ + name: emoji, + count: length(users), + me: !!(current_user && current_user.ap_id in users) + } + end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 7683589cf..61fe81d33 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -337,6 +337,16 @@ defmodule Pleroma.Web.OAuth.OAuthController do ) end + defp handle_token_exchange_error(%Plug.Conn{} = conn, {:account_status, :approval_pending}) do + render_error( + conn, + :forbidden, + "Your account is awaiting approval.", + %{}, + "awaiting_approval" + ) + end + defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do render_invalid_credentials_error(conn) end diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index c8ef3d915..e8a1746d4 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -89,11 +89,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do cm_ref <- MessageReference.for_chat_and_object(chat, message) do conn |> put_view(MessageReferenceView) - |> render("show.json", for: user, chat_message_reference: cm_ref) + |> render("show.json", chat_message_reference: cm_ref) end end - def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{ + def mark_message_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{ id: chat_id, message_id: message_id }) do @@ -104,12 +104,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do {:ok, cm_ref} <- MessageReference.mark_as_read(cm_ref) do conn |> put_view(MessageReferenceView) - |> render("show.json", for: user, chat_message_reference: cm_ref) + |> render("show.json", chat_message_reference: cm_ref) end end def mark_as_read( - %{body_params: %{last_read_id: last_read_id}, assigns: %{user: %{id: user_id}}} = conn, + %{ + body_params: %{last_read_id: last_read_id}, + assigns: %{user: %{id: user_id}} + } = conn, %{id: id} ) do with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id), @@ -121,7 +124,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do end end - def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do + def messages(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id} = params) do with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do cm_refs = chat @@ -130,7 +133,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do conn |> put_view(MessageReferenceView) - |> render("index.json", for: user, chat_message_references: cm_refs) + |> render("index.json", chat_message_references: cm_refs) else _ -> conn diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex index 33ecd1f70..657f46324 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex @@ -21,8 +21,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do ] ) - @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug] - plug(:skip_plug, @skip_plugs when action in [:archive, :show, :list]) + @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug] + plug(:skip_plug, @skip_plugs when action in [:index, :show, :archive]) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaEmojiPackOperation diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex index 1c996da11..04dc20d51 100644 --- a/lib/pleroma/web/pleroma_api/views/chat_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex @@ -15,10 +15,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do def render("show.json", %{chat: %Chat{} = chat} = opts) do recipient = User.get_cached_by_ap_id(chat.recipient) last_message = opts[:last_message] || MessageReference.last_message_for_chat(chat) + account_view_opts = account_view_opts(opts, recipient) %{ id: chat.id |> to_string(), - account: AccountView.render("show.json", Map.put(opts, :user, recipient)), + account: AccountView.render("show.json", account_view_opts), unread: MessageReference.unread_count_for_chat(chat), last_message: last_message && @@ -27,7 +28,17 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do } end - def render("index.json", %{chats: chats}) do - render_many(chats, __MODULE__, "show.json") + def render("index.json", %{chats: chats} = opts) do + render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats)) + end + + defp account_view_opts(opts, recipient) do + account_view_opts = Map.put(opts, :user, recipient) + + if Map.has_key?(account_view_opts, :for) do + account_view_opts + else + Map.put(account_view_opts, :skip_visibility_check, true) + end end end diff --git a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex index 84d2d303d..e0f98b50a 100644 --- a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex +++ b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionView do %{ name: emoji, count: length(users), - accounts: render(AccountView, "index.json", users: users, for: user, as: :user), + accounts: render(AccountView, "index.json", users: users, for: user), me: !!(user && user.ap_id in user_ap_ids) } end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index cdb827e76..16368485e 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.Push.Impl do def build_content( %{ - user: %{notification_settings: %{privacy_option: true}} + user: %{notification_settings: %{hide_notification_contents: true}} } = notification, _actor, _object, diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1729141e9..747f2dc6b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] + validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) page_url - |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> Linkify.Parser.url?(validate_tld: validate_tld) |> parse_uri(page_url) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 386308362..c6433cc53 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -138,6 +138,7 @@ defmodule Pleroma.Web.Router do patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) patch("/users/activate", AdminAPIController, :user_activate) patch("/users/deactivate", AdminAPIController, :user_deactivate) + patch("/users/approve", AdminAPIController, :user_approve) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 750f65386..5ab59b57b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> <div class="input"> <%= label f, :code, "Recovery code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af6e546b0..af85777eb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -10,7 +10,7 @@ <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> <div class="input"> <%= label f, :code, "Authentication code" %> - <%= text_input f, :code %> + <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 5cfb385ac..2294d9d0d 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do |> Map.put(:nickname, params[:username]) |> Map.put(:name, Map.get(params, :fullname, params[:username])) |> Map.put(:password_confirmation, params[:password]) + |> Map.put(:registration_reason, params[:reason]) if Pleroma.Config.get([:instance, :registrations_open]) do create_user(params, opts) @@ -44,6 +45,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do case User.register(changeset) do {:ok, user} -> + maybe_notify_admins(user) {:ok, user} {:error, changeset} -> @@ -56,6 +58,18 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do end end + defp maybe_notify_admins(%User{} = account) do + if Pleroma.Config.get([:instance, :account_approval_required]) do + User.all_superusers() + |> Enum.filter(fn user -> not is_nil(user.email) end) + |> Enum.each(fn superuser -> + superuser + |> Pleroma.Emails.AdminEmail.new_unapproved_registration(account) + |> Pleroma.Emails.Mailer.deliver_async() + end) + end + end + def password_reset(nickname_or_email) do with true <- is_binary(nickname_or_email), %User{local: true, email: email} = user when is_binary(email) <- diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index f739dacb6..b1669d198 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -9,36 +9,6 @@ defmodule Pleroma.Web.MastoFEView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.CustomEmojiView - @default_settings %{ - onboarded: true, - home: %{ - shows: %{ - reblog: true, - reply: true - } - }, - notifications: %{ - alerts: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - }, - shows: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - }, - sounds: %{ - follow: true, - favourite: true, - reblog: true, - mention: true - } - } - } - def initial_state(token, user, custom_emojis) do limit = Config.get([:instance, :limit]) @@ -86,7 +56,7 @@ defmodule Pleroma.Web.MastoFEView do "video\/mp4" ] }, - settings: user.mastofe_settings || @default_settings, + settings: user.mastofe_settings || %{}, push_subscription: nil, accounts: %{user.id => render(AccountView, "show.json", user: user, for: user)}, custom_emojis: render(CustomEmojiView, "index.json", custom_emojis: custom_emojis), |