diff options
Diffstat (limited to 'lib')
70 files changed, 2755 insertions, 641 deletions
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..4dfcc32e7 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -5,6 +5,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do + Mix.Task.run("app.start") Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a4885b70c..dd2b9c8f2 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -74,4 +74,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do inputs: inputs ) end + + def run(["adapters"]) do + start_pleroma() + + :ok = + Pleroma.Gun.Conn.open( + "https://httpbin.org/stream-bytes/1500", + :gun_connections + ) + + Process.sleep(1_500) + + Benchee.run( + %{ + "Without conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool, receive_conn: false] + ) + end, + "Without conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [receive_conn: false] + ) + end, + "With reused conn and without pool" => fn -> + {:ok, %Tesla.Env{}} = + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], + adapter: [pool: :no_pool] + ) + end, + "With reused conn and with pool" => fn -> + {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") + end + }, + parallel: 10 + ) + end end diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 2b03a3009..cdffa88b2 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -4,18 +4,18 @@ defmodule Mix.Tasks.Pleroma.Emoji do use Mix.Task + import Mix.Pleroma @shortdoc "Manages emoji packs" @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md") def run(["ls-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, [], []} = parse_global_opts(args) - manifest = - fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest()) + url_or_path = options[:manifest] || default_manifest() + manifest = fetch_manifest(url_or_path) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -36,14 +36,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do end def run(["get-packs" | args]) do - Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:hackney) + start_pleroma() {options, pack_names, []} = parse_global_opts(args) - manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest() + url_or_path = options[:manifest] || default_manifest() - manifest = fetch_manifest(manifest_url) + manifest = fetch_manifest(url_or_path) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -76,7 +75,10 @@ defmodule Mix.Tasks.Pleroma.Emoji do end # The url specified in files should be in the same directory - files_url = Path.join(Path.dirname(manifest_url), pack["files"]) + files_url = + url_or_path + |> Path.dirname() + |> Path.join(pack["files"]) IO.puts( IO.ANSI.format([ @@ -134,38 +136,51 @@ defmodule Mix.Tasks.Pleroma.Emoji do end end - def run(["gen-pack", src]) do - Application.ensure_all_started(:hackney) + def run(["gen-pack" | args]) do + start_pleroma() + + {opts, [src], []} = + OptionParser.parse( + args, + strict: [ + name: :string, + license: :string, + homepage: :string, + description: :string, + files: :string, + extensions: :string + ] + ) proposed_name = Path.basename(src) |> Path.rootname() - name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) - # If there's no name, use the default one - name = if String.length(name) > 0, do: name, else: proposed_name - - license = String.trim(IO.gets("License: ")) - homepage = String.trim(IO.gets("Homepage: ")) - description = String.trim(IO.gets("Description: ")) + name = get_option(opts, :name, "Pack name:", proposed_name) + license = get_option(opts, :license, "License:") + homepage = get_option(opts, :homepage, "Homepage:") + description = get_option(opts, :description, "Description:") - proposed_files_name = "#{name}.json" - files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: ")) - files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name + proposed_files_name = "#{name}_files.json" + files_name = get_option(opts, :files, "Save file list to:", proposed_files_name) default_exts = [".png", ".gif"] - default_exts_str = Enum.join(default_exts, " ") - exts = - String.trim( - IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ") + custom_exts = + get_option( + opts, + :extensions, + "Emoji file extensions (separated with spaces):", + Enum.join(default_exts, " ") ) + |> String.split(" ", trim: true) exts = - if String.length(exts) > 0 do - String.split(exts, " ") - |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end) - else + if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do default_exts + else + custom_exts end + IO.puts("Using #{Enum.join(exts, " ")} extensions") + IO.puts("Downloading the pack and generating SHA256") binary_archive = Tesla.get!(client(), src).body @@ -195,14 +210,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts(""" #{files_name} has been created and contains the list of all found emojis in the pack. - Please review the files in the remove those not needed. + Please review the files in the pack and remove those not needed. """) - if File.exists?("index.json") do - existing_data = File.read!("index.json") |> Jason.decode!() + pack_file = "#{name}.json" + + if File.exists?(pack_file) do + existing_data = File.read!(pack_file) |> Jason.decode!() File.write!( - "index.json", + pack_file, Jason.encode!( Map.merge( existing_data, @@ -212,11 +229,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) ) - IO.puts("index.json file has been update with the #{name} pack") + IO.puts("#{pack_file} has been updated with the #{name} pack") else - File.write!("index.json", Jason.encode!(pack_json, pretty: true)) + File.write!(pack_file, Jason.encode!(pack_json, pretty: true)) - IO.puts("index.json has been created with the #{name} pack") + IO.puts("#{pack_file} has been created with the #{name} pack") end end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6ca05f74e..5a8329e69 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -95,6 +95,17 @@ defmodule Pleroma.Activity do |> preload([activity, object: object], object: object) end + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) + def user_actor(%Activity{actor: nil}), do: nil + + def user_actor(%Activity{} = activity) do + with %User{} <- activity.user_actor do + activity.user_actor + else + _ -> User.get_cached_by_ap_id(activity.actor) + end + end + def with_joined_user_actor(query, join_type \\ :inner) do join(query, join_type, [activity], u in User, on: u.ap_id == activity.actor, diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 04593b9fb..a34c20343 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Activity.Queries do from(a in query, where: a.actor == ^ap_id) end + def find_by_object_ap_id(activities, object_ap_id) do + Enum.find( + activities, + &(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]]) + ) + end + @spec by_object_id(query, String.t() | [String.t()]) :: query def by_object_id(query \\ Activity, object_id) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 33f1705df..a00938c04 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,8 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do - import Cachex.Spec use Application + + import Cachex.Spec + + alias Pleroma.Config + require Logger @name Mix.Project.config()[:name] @@ -18,9 +22,9 @@ defmodule Pleroma.Application do def repository, do: @repository def user_agent do - case Pleroma.Config.get([:http, :user_agent], :default) do + case Config.get([:http, :user_agent], :default) do :default -> - info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" named_version() <> "; " <> info custom -> @@ -33,27 +37,51 @@ defmodule Pleroma.Application do def start(_type, _args) do Pleroma.Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() - Pleroma.Config.DeprecationWarnings.warn() + Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Repo.check_migrations_applied!() setup_instrumenters() load_custom_modules() + adapter = Application.get_env(:tesla, :adapter) + + if adapter == Tesla.Adapter.Gun do + if version = Pleroma.OTPVersion.version() do + [major, minor] = + version + |> String.split(".") + |> Enum.map(&String.to_integer/1) + |> Enum.take(2) + + if (major == 22 and minor < 2) or major < 22 do + raise " + !!!OTP VERSION WARNING!!! + You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. + " + end + else + raise " + !!!OTP VERSION WARNING!!! + To support correct handling of unordered certificates chains - OTP version must be > 22.2. + " + end + end + # Define workers and child supervisors to be supervised children = [ Pleroma.Repo, - Pleroma.Config.TransferTask, + Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ - hackney_pool_children() ++ + http_children(adapter, @env) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, Pleroma.Config.get(Oban)} + {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -70,7 +98,7 @@ defmodule Pleroma.Application do end def load_custom_modules do - dir = Pleroma.Config.get([:modules, :runtime_dir]) + dir = Config.get([:modules, :runtime_dir]) if dir && File.exists?(dir) do dir @@ -111,20 +139,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -146,7 +160,7 @@ defmodule Pleroma.Application do do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), do: %{ @@ -155,9 +169,9 @@ defmodule Pleroma.Application do type: :worker } - defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp chat_enabled?, do: Config.get([:chat, :enabled]) - defp streamer_child(:test), do: [] + defp streamer_child(env) when env in [:test, :benchmark], do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] @@ -169,13 +183,6 @@ defmodule Pleroma.Application do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) - end - end - defp task_children(:test) do [ %{ @@ -200,4 +207,31 @@ defmodule Pleroma.Application do } ] end + + # start hackney and gun pools in tests + defp http_children(_, :test) do + hackney_options = Config.get([:hackney_pools, :federation]) + hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) + [hackney_pool, Pleroma.Pool.Supervisor] + end + + defp http_children(Tesla.Adapter.Hackney, _) do + pools = [:federation, :media] + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) do + [:upload | pools] + else + pools + end + + for pool <- pools do + options = Config.get([:hackney_pools, pool]) + :hackney_pool.child_spec(pool, options) + end + end + + defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] + + defp http_children(_, _), do: [] end diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 2b43d4c36..4097ee5b7 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -278,8 +278,6 @@ defmodule Pleroma.ConfigDB do } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do value = entity @@ -323,15 +321,6 @@ defmodule Pleroma.ConfigDB do {:proxy_url, {do_transform_string(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = - entity - |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") - |> Code.eval_string() - - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 7c3449b5e..936bc9ab1 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Config.TransferTask do use Task + alias Pleroma.Config alias Pleroma.ConfigDB alias Pleroma.Repo @@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Oban}, {:pleroma, :rate_limit}, {:pleroma, :markup}, - {:plerome, :streamer} + {:pleroma, :streamer}, + {:pleroma, :pools}, + {:pleroma, :connections_pool} ] @reboot_time_subkeys [ @@ -32,45 +35,33 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :gopher, [:enabled]} ] - @reject [nil, :prometheus] - def start_link(_) do load_and_update_env() - if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) + if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end - @spec load_and_update_env([ConfigDB.t()]) :: :ok | false - def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do - with {:configurable, true} <- - {:configurable, Pleroma.Config.get(:configurable_from_database)}, - true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), - started_applications <- Application.started_applications() do + @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok + def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do + with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - in_db = Repo.all(ConfigDB) - - with_deleted = in_db ++ deleted - - reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject] - - applications = - with_deleted - |> Enum.map(&merge_and_update(&1)) - |> Enum.uniq() - # TODO: some problem with prometheus after restart! - |> Enum.reject(&(&1 in reject_for_restart)) - - # to be ensured that pleroma will be restarted last - applications = - if :pleroma in applications do - List.delete(applications, :pleroma) ++ [:pleroma] + # TODO: some problem with prometheus after restart! + reject_restart = + if restart_pleroma? do + [nil, :prometheus] else - Restarter.Pleroma.rebooted() - applications + [:pleroma, nil, :prometheus] end - Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) + started_applications = Application.started_applications() + + (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.map(&merge_and_update/1) + |> Enum.uniq() + |> Enum.reject(&(&1 in reject_restart)) + |> maybe_set_pleroma_last() + |> Enum.each(&restart(started_applications, &1, Config.get(:env))) :ok else @@ -78,42 +69,54 @@ defmodule Pleroma.Config.TransferTask do end end + defp maybe_set_pleroma_last(apps) do + # to be ensured that pleroma will be restarted last + if :pleroma in apps do + apps + |> List.delete(:pleroma) + |> List.insert_at(-1, :pleroma) + else + Restarter.Pleroma.rebooted() + apps + end + end + + defp group_for_restart(:logger, key, _, merged_value) do + # change logger configuration in runtime, without restart + if Keyword.keyword?(merged_value) and + key not in [:compile_time_application, :backends, :compile_time_purge_matching] do + Logger.configure_backend(key, merged_value) + else + Logger.configure([{key, merged_value}]) + end + + nil + end + + defp group_for_restart(group, _, _, _) when group != :pleroma, do: group + + defp group_for_restart(group, key, value, _) do + if pleroma_need_restart?(group, key, value), do: group + end + defp merge_and_update(setting) do try do key = ConfigDB.from_string(setting.key) group = ConfigDB.from_string(setting.group) - default = Pleroma.Config.Holder.default_config(group, key) + default = Config.Holder.default_config(group, key) value = ConfigDB.from_binary(setting.value) merged_value = - if Ecto.get_meta(setting, :state) == :deleted do - default - else - if can_be_merged?(default, value) do - ConfigDB.merge_group(group, key, default, value) - else - value - end + cond do + Ecto.get_meta(setting, :state) == :deleted -> default + can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) + true -> value end :ok = update_env(group, key, merged_value) - if group != :logger do - if group != :pleroma or pleroma_need_restart?(group, key, value) do - group - end - else - # change logger configuration in runtime, without restart - if Keyword.keyword?(merged_value) and - key not in [:compile_time_application, :backends, :compile_time_purge_matching] do - Logger.configure_backend(key, merged_value) - else - Logger.configure([{key, merged_value}]) - end - - nil - end + group_for_restart(group, key, value, merged_value) rescue error -> error_msg = diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 693825cf5..215265fc9 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -129,21 +129,18 @@ defmodule Pleroma.Conversation.Participation do end def restrict_recipients(query, user, %{"recipients" => user_ids}) do - user_ids = + user_binary_ids = [user.id | user_ids] |> Enum.uniq() - |> Enum.reduce([], fn user_id, acc -> - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) - [user_id | acc] - end) + |> User.binary_id() conversation_subquery = __MODULE__ |> group_by([p], p.conversation_id) |> having( [p], - count(p.user_id) == ^length(user_ids) and - fragment("array_agg(?) @> ?", p.user_id, ^user_ids) + count(p.user_id) == ^length(user_binary_ids) and + fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids) ) |> select([p], %{id: p.conversation_id}) diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index a6d281151..a9538ea4e 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -129,4 +129,32 @@ defmodule Pleroma.FollowingRelationship do move_following(origin, target) end end + + def all_between_user_sets( + source_users, + target_users + ) + when is_list(source_users) and is_list(target_users) do + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) + + __MODULE__ + |> where( + fragment( + "(follower_id = ANY(?) AND following_id = ANY(?)) OR \ + (follower_id = ANY(?) AND following_id = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^target_user_ids, + ^source_user_ids + ) + ) + |> Repo.all() + end + + def find(following_relationships, follower, following) do + Enum.find(following_relationships, fn + fr -> fr.follower_id == follower.id and fr.following_id == following.id + end) + end end diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex new file mode 100644 index 000000000..f51cd7db8 --- /dev/null +++ b/lib/pleroma/gun/api.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @behaviour Pleroma.Gun + + alias Pleroma.Gun + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :socks_opts, + :ws_opts + ] + + @impl Gun + def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys)) + + @impl Gun + defdelegate info(pid), to: :gun + + @impl Gun + defdelegate close(pid), to: :gun + + @impl Gun + defdelegate await_up(pid, timeout \\ 5_000), to: :gun + + @impl Gun + defdelegate connect(pid, opts), to: :gun + + @impl Gun + defdelegate await(pid, ref), to: :gun + + @impl Gun + defdelegate set_owner(pid, owner), to: :gun +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..20823a765 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,196 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + alias Pleroma.Gun + alias Pleroma.Pool.Connections + + require Logger + + @type gun_state :: :up | :down + @type conn_state :: :active | :idle + + @type t :: %__MODULE__{ + conn: pid(), + gun_state: gun_state(), + conn_state: conn_state(), + used_by: [pid()], + last_reference: pos_integer(), + crf: float(), + retries: pos_integer() + } + + defstruct conn: nil, + gun_state: :open, + conn_state: :init, + used_by: [], + last_reference: 0, + crf: 1, + retries: 0 + + @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil + def open(url, name, opts \\ []) + def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) + + def open(%URI{} = uri, name, opts) do + pool_opts = Pleroma.Config.get([:connections_pool], []) + + opts = + opts + |> Enum.into(%{}) + |> Map.put_new(:retry, pool_opts[:retry] || 1) + |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) + |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> maybe_add_tls_opts(uri) + + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + conn_pid = + if Connections.count(name) < opts[:max_connection] do + do_open(uri, opts) + else + close_least_used_and_do_open(name, uri, opts) + end + + if is_pid(conn_pid) do + conn = %Pleroma.Gun.Conn{ + conn: conn_pid, + gun_state: :up, + conn_state: :active, + last_reference: :os.system_time(:second) + } + + :ok = Gun.set_owner(conn_pid, Process.whereis(name)) + Connections.add_conn(name, key, conn) + end + end + + defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts + + defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + tls_opts = [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + reuse_sessions: false, + verify_fun: + {&:ssl_verify_hostname.verify_fun/3, + [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + ] + + tls_opts = + if Keyword.keyword?(opts[:tls_opts]) do + Keyword.merge(tls_opts, opts[:tls_opts]) + else + tls_opts + end + + Map.put(opts, :tls_opts, tls_opts) + end + + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do + connect_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + stream <- Gun.connect(conn, connect_opts), + {:response, :fin, 200, _} <- Gun.await(conn, stream) do + conn + else + error -> + Logger.warn( + "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" + ) + + error + end + end + + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do + version = + proxy_type + |> to_string() + |> String.last() + |> case do + "4" -> 4 + _ -> 5 + end + + socks_opts = + uri + |> destination_opts() + |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> Map.put(:version, version) + + opts = + opts + |> Map.put(:protocols, [:socks]) + |> Map.put(:socks_opts, socks_opts) + + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{ + inspect(error) + }" + ) + + error + end + end + + defp do_open(%URI{host: host, port: port} = uri, opts) do + host = Pleroma.HTTP.Connection.parse_host(host) + + with {:ok, conn} <- Gun.open(host, port, opts), + {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do + conn + else + error -> + Logger.warn( + "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" + ) + + error + end + end + + defp destination_opts(%URI{host: host, port: port}) do + host = Pleroma.HTTP.Connection.parse_host(host) + %{host: host, port: port} + end + + defp add_http2_opts(opts, "https", tls_opts) do + Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + end + + defp add_http2_opts(opts, _, _), do: opts + + defp close_least_used_and_do_open(name, uri, opts) do + with [{key, conn} | _conns] <- Connections.get_unused_conns(name), + :ok <- Gun.close(conn.conn) do + Connections.remove_conn(name, key) + + do_open(uri, opts) + else + [] -> {:error, :pool_overflowed} + end + end + + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do + "#{scheme}://#{host}#{path}" + end +end diff --git a/lib/pleroma/gun/gun.ex b/lib/pleroma/gun/gun.ex new file mode 100644 index 000000000..4043e4880 --- /dev/null +++ b/lib/pleroma/gun/gun.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback set_owner(pid(), pid()) :: :ok + + @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) + + defp api, do: @api + + def open(host, port, opts), do: api().open(host, port, opts) + + def info(pid), do: api().info(pid) + + def close(pid), do: api().close(pid) + + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + def set_owner(pid, owner), do: api().set_owner(pid, owner) +end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex new file mode 100644 index 000000000..510722ff9 --- /dev/null +++ b/lib/pleroma/http/adapter_helper.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper do + alias Pleroma.HTTP.Connection + + @type proxy :: + {Connection.host(), pos_integer()} + | {Connection.proxy_type(), Connection.host(), pos_integer()} + + @callback options(keyword(), URI.t()) :: keyword() + @callback after_request(keyword()) :: :ok + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + maybe_add_proxy(opts, format_proxy(proxy)) + end + + @spec maybe_get_conn(URI.t(), keyword()) :: keyword() + def maybe_get_conn(_uri, opts), do: opts + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil + def format_proxy(nil), do: nil + + def format_proxy(proxy_url) do + case Connection.parse_proxy(proxy_url) do + {:ok, host, port} -> {host, port} + {:ok, type, host, port} -> {type, host, port} + _ -> nil + end + end + + @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() + def maybe_add_proxy(opts, nil), do: opts + def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex new file mode 100644 index 000000000..ead7cdc6b --- /dev/null +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.AdapterHelper.Gun do + @behaviour Pleroma.HTTP.AdapterHelper + + alias Pleroma.HTTP.AdapterHelper + alias Pleroma.Pool.Connections + + require Logger + + @defaults [ + connect_timeout: 5_000, + domain_lookup_timeout: 5_000, + tls_handshake_timeout: 5_000, + retry: 1, + retry_timeout: 1000, + await_up_timeout: 5_000 + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(incoming_opts \\ [], %URI{} = uri) do + proxy = + Pleroma.Config.get([:http, :proxy_url]) + |> AdapterHelper.format_proxy() + + config_opts = Pleroma.Config.get([:http, :adapter], []) + + @defaults + |> Keyword.merge(config_opts) + |> add_scheme_opts(uri) + |> AdapterHelper.maybe_add_proxy(proxy) + |> maybe_get_conn(uri, incoming_opts) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts) do + if opts[:conn] && opts[:body_as] != :chunks do + Connections.checkout(opts[:conn], self(), :gun_connections) + end + + :ok + end + + defp add_scheme_opts(opts, %{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %{scheme: "https"}) do + opts + |> Keyword.put(:certificates_verification, true) + |> Keyword.put(:tls_opts, log_level: :warning) + end + + defp maybe_get_conn(adapter_opts, uri, incoming_opts) do + {receive_conn?, opts} = + adapter_opts + |> Keyword.merge(incoming_opts) + |> Keyword.pop(:receive_conn, true) + + if Connections.alive?(:gun_connections) and receive_conn? do + checkin_conn(uri, opts) + else + opts + end + end + + defp checkin_conn(uri, opts) do + case Connections.checkin(uri, :gun_connections) do + nil -> + Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) + opts + + conn when is_pid(conn) -> + Keyword.merge(opts, conn: conn, close_conn: false) + end + end +end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex new file mode 100644 index 000000000..dcb4cac71 --- /dev/null +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -0,0 +1,43 @@ +defmodule Pleroma.HTTP.AdapterHelper.Hackney do + @behaviour Pleroma.HTTP.AdapterHelper + + @defaults [ + connect_timeout: 10_000, + recv_timeout: 20_000, + follow_redirect: true, + force_redirect: true, + pool: :federation + ] + + @spec options(keyword(), URI.t()) :: keyword() + def options(connection_opts \\ [], %URI{} = uri) do + proxy = Pleroma.Config.get([:http, :proxy_url]) + + config_opts = Pleroma.Config.get([:http, :adapter], []) + + @defaults + |> Keyword.merge(config_opts) + |> Keyword.merge(connection_opts) + |> add_scheme_opts(uri) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) + end + + defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts + + defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do + ssl_opts = [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"], + server_name_indication: to_charlist(host) + ] + ] + + Keyword.merge(opts, ssl_opts) + end + + def after_request(_), do: :ok +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 80e6c30d6..ebacf7902 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -4,40 +4,121 @@ defmodule Pleroma.HTTP.Connection do @moduledoc """ - Connection for http-requests. + Configure Tesla.Client with default and customized adapter options. """ - @hackney_options [ - connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation - ] - @adapter Application.get_env(:tesla, :adapter) + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper - @doc """ - Configure a client connection + require Logger + + @defaults [pool: :federation] - # Returns + @type ip_address :: ipv4_address() | ipv6_address() + @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} + @type ipv6_address :: + {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | ip_address() - Tesla.Env.client + @doc """ + Merge default connection & adapter options with received ones. """ - @spec new(Keyword.t()) :: Tesla.Env.client() - def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> pool_timeout() + |> Keyword.merge(opts) + |> adapter_helper().options(uri) + end + + defp pool_timeout(opts) do + {config_key, default} = + if adapter() == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout])} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) + + Keyword.merge(opts, timeout: timeout) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts), do: adapter_helper().after_request(opts) + + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + :error -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end end - # fetch Hackney options - # - def hackney_options(opts) do - options = Keyword.get(opts, :adapter, []) - adapter_options = Pleroma.Config.get([:http, :adapter], []) - proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - - @hackney_options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ee5b5e127..583b56484 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -4,21 +4,47 @@ defmodule Pleroma.HTTP do @moduledoc """ - + Wrapper for `Tesla.request/2`. """ alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder + alias Tesla.Client + alias Tesla.Env + + require Logger @type t :: __MODULE__ @doc """ - Builds and perform http request. + Performs GET request. + + See `Pleroma.HTTP.request/5` + """ + @spec get(Request.url() | nil, Request.headers(), keyword()) :: + nil | {:ok, Env.t()} | {:error, any()} + def get(url, headers \\ [], options \\ []) + def get(nil, _, _), do: nil + def get(url, headers, options), do: request(:get, url, "", headers, options) + + @doc """ + Performs POST request. + + See `Pleroma.HTTP.request/5` + """ + @spec post(Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def post(url, body, headers \\ [], options \\ []), + do: request(:post, url, body, headers, options) + + @doc """ + Builds and performs http request. # Arguments: `method` - :get, :post, :put, :delete - `url` - `body` + `url` - full url + `body` - request body `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` `options` - custom, per-request middleware or adapter options @@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - def request(method, url, body \\ "", headers \\ [], options \\ []) do - try do - options = - process_request_options(options) - |> process_sni_options(url) - - params = Keyword.get(options, :params, []) - - %{} - |> Builder.method(method) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.url(url) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() - rescue - e -> - {:error, e} - catch - :exit, e -> - {:error, e} - end - end + @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) :: + {:ok, Env.t()} | {:error, any()} + def request(method, url, body, headers, options) when is_binary(url) do + uri = URI.parse(url) + adapter_opts = Connection.options(uri, options[:adapter] || []) + options = put_in(options[:adapter], adapter_opts) + params = options[:params] || [] + request = build_request(method, headers, options, url, body, params) - defp process_sni_options(options, nil), do: options + adapter = Application.get_env(:tesla, :adapter) + client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() + pid = Process.whereis(adapter_opts[:pool]) - case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options - end - end + pool_alive? = + if adapter == Tesla.Adapter.Gun && pid do + Process.alive?(pid) + else + false + end + + request_opts = + adapter_opts + |> Enum.into(%{}) + |> Map.put(:env, Pleroma.Config.get([:env])) + |> Map.put(:pool_alive?, pool_alive?) + + response = request(client, request, request_opts) + + Connection.after_request(adapter_opts) - def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + response end - @doc """ - Performs GET request. + @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} + def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - See `Pleroma.HTTP.request/5` - """ - def get(url, headers \\ [], options \\ []), - do: request(:get, url, "", headers, options) + def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - @doc """ - Performs POST request. + def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) - See `Pleroma.HTTP.request/5` - """ - def post(url, body, headers \\ [], options \\ []), - do: request(:post, url, body, headers, options) + def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do + :poolboy.transaction( + pool, + &Pleroma.Pool.Request.execute(&1, client, request, timeout), + timeout + ) + end + + @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} + def request(client, request), do: Tesla.request(client, request) + + defp build_request(method, headers, options, url, body, params) do + Builder.new() + |> Builder.method(method) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.url(url) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Builder.convert_to_keyword() + end end diff --git a/lib/pleroma/http/request.ex b/lib/pleroma/http/request.ex new file mode 100644 index 000000000..761bd6ccf --- /dev/null +++ b/lib/pleroma/http/request.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Request do + @moduledoc """ + Request struct. + """ + defstruct method: :get, url: "", query: [], headers: [], body: "", opts: [] + + @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch + @type url :: String.t() + @type headers :: [{String.t(), String.t()}] + + @type t :: %__MODULE__{ + method: method(), + url: url(), + query: keyword(), + headers: headers(), + body: String.t(), + opts: keyword() + } +end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 77ef4bfd8..2fc876d92 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do Helper functions for building Tesla requests """ - @doc """ - Specify the request method when building a request - - ## Parameters - - - request (Map) - Collected request options - - m (atom) - Request method - - ## Returns + alias Pleroma.HTTP.Request + alias Tesla.Multipart - Map + @doc """ + Creates new request """ - @spec method(map(), atom) :: map() - def method(request, m) do - Map.put_new(request, :method, m) - end + @spec new(Request.t()) :: Request.t() + def new(%Request{} = request \\ %Request{}), do: request @doc """ Specify the request method when building a request + """ + @spec method(Request.t(), Request.method()) :: Request.t() + def method(request, m), do: %{request | method: m} - ## Parameters - - - request (Map) - Collected request options - - u (String) - Request URL - - ## Returns - - Map + @doc """ + Specify the request method when building a request """ - @spec url(map(), String.t()) :: map() - def url(request, u) do - Map.put_new(request, :url, u) - end + @spec url(Request.t(), Request.url()) :: Request.t() + def url(request, u), do: %{request | url: u} @doc """ Add headers to the request """ - @spec headers(map(), list(tuple)) :: map() - def headers(request, header_list) do - header_list = + @spec headers(Request.t(), Request.headers()) :: Request.t() + def headers(request, headers) do + headers_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + [{"user-agent", Pleroma.Application.user_agent()} | headers] else - header_list + headers end - Map.put_new(request, :headers, header_list) + %{request | headers: headers_list} end @doc """ Add custom, per-request middleware or adapter options to the request """ - @spec opts(map(), Keyword.t()) :: map() - def opts(request, options) do - Map.put_new(request, :opts, options) - end - - @doc """ - Add optional parameters to the request - - ## Parameters - - - request (Map) - Collected request options - - definitions (Map) - Map of parameter name to parameter location. - - options (KeywordList) - The provided optional parameters - - ## Returns - - Map - """ - @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map() - def add_optional_params(request, _, []), do: request - - def add_optional_params(request, definitions, [{key, value} | tail]) do - case definitions do - %{^key => location} -> - request - |> add_param(location, key, value) - |> add_optional_params(definitions, tail) - - _ -> - add_optional_params(request, definitions, tail) - end - end + @spec opts(Request.t(), keyword()) :: Request.t() + def opts(request, options), do: %{request | opts: options} @doc """ Add optional parameters to the request - - ## Parameters - - - request (Map) - Collected request options - - location (atom) - Where to put the parameter - - key (atom) - The name of the parameter - - value (any) - The value of the parameter - - ## Returns - - Map """ - @spec add_param(map(), atom, atom, any()) :: map() - def add_param(request, :query, :query, values), do: Map.put(request, :query, values) + @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t() + def add_param(request, :query, :query, values), do: %{request | query: values} - def add_param(request, :body, :body, value), do: Map.put(request, :body, value) + def add_param(request, :body, :body, value), do: %{request | body: value} def add_param(request, :body, key, value) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) + |> Map.put(:body, Multipart.new()) |> Map.update!( :body, - &Tesla.Multipart.add_field( + &Multipart.add_field( &1, key, Jason.encode!(value), - headers: [{:"Content-Type", "application/json"}] + headers: [{"content-type", "application/json"}] ) ) end def add_param(request, :file, name, path) do request - |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) - |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name)) + |> Map.put(:body, Multipart.new()) + |> Map.update!(:body, &Multipart.add_file(&1, path, name: name)) end def add_param(request, :form, name, value) do - request - |> Map.update(:body, %{name => value}, &Map.put(&1, name, value)) + Map.update(request, :body, %{name => value}, &Map.put(&1, name, value)) end def add_param(request, location, key, value) do Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}])) end + + def convert_to_keyword(request) do + request + |> Map.from_struct() + |> Enum.into([]) + end end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 3ef3b3f58..04ee510b9 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Notification do alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Push @@ -17,6 +18,7 @@ defmodule Pleroma.Notification do import Ecto.Query import Ecto.Changeset + require Logger @type t :: %__MODULE__{} @@ -37,11 +39,11 @@ defmodule Pleroma.Notification do end defp for_user_query_ap_id_opts(user, opts) do - ap_id_relations = + ap_id_relationships = [:block] ++ if opts[@include_muted_option], do: [], else: [:notification_mute] - preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships) exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts) @@ -100,7 +102,7 @@ defmodule Pleroma.Notification do query |> where([n, a], a.actor not in ^notification_muted_ap_ids) - |> join(:left, [n, a], tm in Pleroma.ThreadMute, + |> join(:left, [n, a], tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) ) |> where([n, a, o, tm], is_nil(tm.user_id)) @@ -275,58 +277,111 @@ defmodule Pleroma.Notification do def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity) - unless object && object.data["type"] == "Answer" do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) - {:ok, notifications} - else + if object && object.data["type"] == "Answer" do {:ok, []} + else + do_create_notifications(activity) end end def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do + do_create_notifications(activity) + end + + def create_notifications(_), do: {:ok, []} + + defp do_create_notifications(%Activity{} = activity) do + {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) + potential_receivers = enabled_receivers ++ disabled_receivers + notifications = - activity - |> get_notified_from_activity() - |> Enum.map(&create_notification(activity, &1)) + Enum.map(potential_receivers, fn user -> + do_send = user in enabled_receivers + create_notification(activity, user, do_send) + end) {:ok, notifications} end - def create_notifications(_), do: {:ok, []} - # TODO move to sql, too. - def create_notification(%Activity{} = activity, %User{} = user) do + def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - ["user", "user:notification"] - |> Streamer.stream(notification) + if do_send do + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end - Push.send(notification) notification end end + @doc """ + Returns a tuple with 2 elements: + {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)} + + NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1 + """ def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do - [] - |> Utils.maybe_notify_to_recipients(activity) - |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) - |> Utils.maybe_notify_followers(activity) - |> Enum.uniq() - |> User.get_users_from_set(local_only) + potential_receiver_ap_ids = + [] + |> Utils.maybe_notify_to_recipients(activity) + |> Utils.maybe_notify_mentioned_recipients(activity) + |> Utils.maybe_notify_subscribers(activity) + |> Utils.maybe_notify_followers(activity) + |> Enum.uniq() + + # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs + notification_enabled_ap_ids = + potential_receiver_ap_ids + |> exclude_relationship_restricted_ap_ids(activity) + |> exclude_thread_muter_ap_ids(activity) + + potential_receivers = + potential_receiver_ap_ids + |> Enum.uniq() + |> User.get_users_from_set(local_only) + + notification_enabled_users = + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + + {notification_enabled_users, potential_receivers -- notification_enabled_users} + end + + def get_notified_from_activity(_, _local_only), do: {[], []} + + @doc "Filters out AP IDs of users basing on their relationships with activity actor user" + def exclude_relationship_restricted_ap_ids([], _activity), do: [] + + def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do + relationship_restricted_ap_ids = + activity + |> Activity.user_actor() + |> User.incoming_relationships_ungrouped_ap_ids([ + :block, + :notification_mute + ]) + + Enum.uniq(ap_ids) -- relationship_restricted_ap_ids end - def get_notified_from_activity(_, _local_only), do: [] + @doc "Filters out AP IDs of users who mute activity thread" + def exclude_thread_muter_ap_ids([], _activity), do: [] + + def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do + thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"]) + + Enum.uniq(ap_ids) -- thread_muter_ap_ids + end @spec skip?(Activity.t(), User.t()) :: boolean() - def skip?(activity, user) do + def skip?(%Activity{} = activity, %User{} = user) do [ :self, :followers, @@ -335,18 +390,20 @@ defmodule Pleroma.Notification do :non_follows, :recently_followed ] - |> Enum.any?(&skip?(&1, activity, user)) + |> Enum.find(&skip?(&1, activity, user)) end + def skip?(_, _), do: false + @spec skip?(atom(), Activity.t(), User.t()) :: boolean() - def skip?(:self, activity, user) do + def skip?(:self, %Activity{} = activity, %User{} = user) do activity.data["actor"] == user.ap_id end def skip?( :followers, - activity, - %{notification_settings: %{followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) @@ -355,15 +412,19 @@ defmodule Pleroma.Notification do def skip?( :non_followers, - activity, - %{notification_settings: %{non_followers: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do + def skip?( + :follows, + %Activity{} = activity, + %User{notification_settings: %{follows: false}} = user + ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) @@ -371,15 +432,16 @@ defmodule Pleroma.Notification do def skip?( :non_follows, - activity, - %{notification_settings: %{non_follows: false}} = user + %Activity{} = activity, + %User{notification_settings: %{non_follows: false}} = user ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) !User.following?(user, followed) end - def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do + # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL + def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] Notification.for_user(user) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 9ae6a5600..99608b8a5 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -32,6 +32,18 @@ defmodule Pleroma.Object.Containment do get_actor(%{"actor" => actor}) end + def get_object(%{"object" => id}) when is_binary(id) do + id + end + + def get_object(%{"object" => %{"id" => id}}) when is_binary(id) do + id + end + + def get_object(_) do + nil + end + # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus # objects being present in the test suite environment. Once these objects are # removed, please also remove this. diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index eaa13d1e7..263ded5dd 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -154,7 +154,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -166,7 +166,7 @@ defmodule Pleroma.Object.Fetcher do date = Pleroma.Signature.signed_date() headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/otp_version.ex b/lib/pleroma/otp_version.ex new file mode 100644 index 000000000..114d0054f --- /dev/null +++ b/lib/pleroma/otp_version.ex @@ -0,0 +1,28 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.OTPVersion do + @spec version() :: String.t() | nil + def version do + # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version + [ + Path.join(:code.root_dir(), "OTP_VERSION"), + Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"]) + ] + |> get_version_from_files() + end + + @spec get_version_from_files([Path.t()]) :: String.t() | nil + def get_version_from_files([]), do: nil + + def get_version_from_files([path | paths]) do + if File.exists?(path) do + path + |> File.read!() + |> String.replace(~r/\r|\n|\s/, "") + else + get_version_from_files(paths) + end + end +end diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex new file mode 100644 index 000000000..4d4ba913c --- /dev/null +++ b/lib/pleroma/pool/connections.ex @@ -0,0 +1,283 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Connections do + use GenServer + + alias Pleroma.Config + alias Pleroma.Gun + + require Logger + + @type domain :: String.t() + @type conn :: Pleroma.Gun.Conn.t() + + @type t :: %__MODULE__{ + conns: %{domain() => conn()}, + opts: keyword() + } + + defstruct conns: %{}, opts: [] + + @spec start_link({atom(), keyword()}) :: {:ok, pid()} + def start_link({name, opts}) do + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + + @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil + def checkin(url, name) + def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) + + def checkin(%URI{} = uri, name) do + timeout = Config.get([:connections_pool, :checkin_timeout], 250) + + GenServer.call(name, {:checkin, uri}, timeout) + end + + @spec alive?(atom()) :: boolean() + def alive?(name) do + if pid = Process.whereis(name) do + Process.alive?(pid) + else + false + end + end + + @spec get_state(atom()) :: t() + def get_state(name) do + GenServer.call(name, :state) + end + + @spec count(atom()) :: pos_integer() + def count(name) do + GenServer.call(name, :count) + end + + @spec get_unused_conns(atom()) :: [{domain(), conn()}] + def get_unused_conns(name) do + GenServer.call(name, :unused_conns) + end + + @spec checkout(pid(), pid(), atom()) :: :ok + def checkout(conn, pid, name) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok + def add_conn(name, key, conn) do + GenServer.cast(name, {:add_conn, key, conn}) + end + + @spec remove_conn(atom(), String.t()) :: :ok + def remove_conn(name, key) do + GenServer.cast(name, {:remove_conn, key}) + end + + @impl true + def handle_cast({:add_conn, key, conn}, state) do + state = put_in(state.conns[key], conn) + + Process.monitor(conn.conn) + {:noreply, state} + end + + @impl true + def handle_cast({:checkout, conn_pid, pid}, state) do + state = + with true <- Process.alive?(conn_pid), + {key, conn} <- find_conn(state.conns, conn_pid), + used_by <- List.keydelete(conn.used_by, pid, 0) do + conn_state = if used_by == [], do: :idle, else: conn.conn_state + + put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + else + false -> + Logger.debug("checkout for closed conn #{inspect(conn_pid)}") + state + + nil -> + Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") + state + end + + {:noreply, state} + end + + @impl true + def handle_cast({:remove_conn, key}, state) do + state = put_in(state.conns, Map.delete(state.conns, key)) + {:noreply, state} + end + + @impl true + def handle_call({:checkin, uri}, from, state) do + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + case state.conns[key] do + %{conn: pid, gun_state: :up} = conn -> + time = :os.system_time(:second) + last_reference = time - conn.last_reference + crf = crf(last_reference, 100, conn.crf) + + state = + put_in(state.conns[key], %{ + conn + | last_reference: time, + crf: crf, + conn_state: :active, + used_by: [from | conn.used_by] + }) + + {:reply, pid, state} + + %{gun_state: :down} -> + {:reply, nil, state} + + nil -> + {:reply, nil, state} + end + end + + @impl true + def handle_call(:state, _from, state), do: {:reply, state, state} + + @impl true + def handle_call(:count, _from, state) do + {:reply, Enum.count(state.conns), state} + end + + @impl true + def handle_call(:unused_conns, _from, state) do + unused_conns = + state.conns + |> Enum.filter(&filter_conns/1) + |> Enum.sort(&sort_conns/2) + + {:reply, unused_conns, state} + end + + defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true + defp filter_conns(_), do: false + + defp sort_conns({_, c1}, {_, c2}) do + c1.crf <= c2.crf and c1.last_reference <= c2.last_reference + end + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do + %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) + + host = + case :inet.ntoa(host) do + {:error, :einval} -> host + ip -> ip + end + + key = "#{scheme}:#{host}:#{port}" + + state = + with {key, conn} <- find_conn(state.conns, conn_pid, key), + {true, key} <- {Process.alive?(conn_pid), key} do + put_in(state.conns[key], %{ + conn + | gun_state: :up, + conn_state: :active, + retries: 0 + }) + else + {false, key} -> + put_in( + state.conns, + Map.delete(state.conns, key) + ) + + nil -> + :ok = Gun.close(conn_pid) + + state + end + + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do + retries = Config.get([:connections_pool, :retry], 1) + # we can't get info on this pid, because pid is dead + state = + with {key, conn} <- find_conn(state.conns, conn_pid), + {true, key} <- {Process.alive?(conn_pid), key} do + if conn.retries == retries do + :ok = Gun.close(conn.conn) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + put_in(state.conns[key], %{ + conn + | gun_state: :down, + retries: conn.retries + 1 + }) + end + else + {false, key} -> + put_in( + state.conns, + Map.delete(state.conns, key) + ) + + nil -> + Logger.debug(":gun_down for conn which isn't found in state") + + state + end + + {:noreply, state} + end + + @impl true + def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do + Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") + + state = + with {key, conn} <- find_conn(state.conns, conn_pid) do + Enum.each(conn.used_by, fn {pid, _ref} -> + Process.exit(pid, reason) + end) + + put_in( + state.conns, + Map.delete(state.conns, key) + ) + else + nil -> + Logger.debug(":DOWN for conn which isn't found in state") + + state + end + + {:noreply, state} + end + + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + + defp find_conn(conns, conn_pid, conn_key) do + Enum.find(conns, fn {key, conn} -> + key == conn_key and conn.conn == conn_pid + end) + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end +end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex new file mode 100644 index 000000000..21a6fbbc5 --- /dev/null +++ b/lib/pleroma/pool/pool.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool do + def child_spec(opts) do + poolboy_opts = + opts + |> Keyword.put(:worker_module, Pleroma.Pool.Request) + |> Keyword.put(:name, {:local, opts[:name]}) + |> Keyword.put(:size, opts[:size]) + |> Keyword.put(:max_overflow, opts[:max_overflow]) + + %{ + id: opts[:id] || {__MODULE__, make_ref()}, + start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, + restart: :permanent, + shutdown: 5000, + type: :worker + } + end +end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex new file mode 100644 index 000000000..3fb930db7 --- /dev/null +++ b/lib/pleroma/pool/request.ex @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Request do + use GenServer + + require Logger + + def start_link(args) do + GenServer.start_link(__MODULE__, args) + end + + @impl true + def init(_), do: {:ok, []} + + @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: + {:ok, Tesla.Env.t()} | {:error, any()} + def execute(pid, client, request, timeout) do + GenServer.call(pid, {:execute, client, request}, timeout) + end + + @impl true + def handle_call({:execute, client, request}, _from, state) do + response = Pleroma.HTTP.request(client, request) + + {:reply, response, state} + end + + @impl true + def handle_info({:gun_data, _conn, _stream, _, _}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_up, _conn, _protocol}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_error, _conn, _stream, _error}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do + {:noreply, state} + end + + @impl true + def handle_info(msg, state) do + Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") + {:noreply, state} + end +end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex new file mode 100644 index 000000000..faf646cb2 --- /dev/null +++ b/lib/pleroma/pool/supervisor.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Pool.Supervisor do + use Supervisor + + alias Pleroma.Config + alias Pleroma.Pool + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def init(_) do + conns_child = %{ + id: Pool.Connections, + start: + {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} + } + + Supervisor.init([conns_child | pools()], strategy: :one_for_one) + end + + defp pools do + pools = Config.get(:pools) + + pools = + if Config.get([Pleroma.Upload, :proxy_remote]) == false do + Keyword.delete(pools, :upload) + else + pools + end + + for {pool_name, pool_opts} <- pools do + pool_opts + |> Keyword.put(:id, {Pool, pool_name}) + |> Keyword.put(:name, pool_name) + |> Pool.child_spec() + end + end +end diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 26d14fabd..0d13ff174 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,19 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} - @callback stream_body(reference() | pid() | map()) :: - {:ok, binary()} | :done | {:error, String.t()} + @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()} @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + :tesla + |> Application.get_env(:adapter) + |> client() end + + defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney + defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla + defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client) end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e84118a90 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + @impl true + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + @impl true + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + @impl true + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..e81ea8bde --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,90 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + + @type headers() :: [{String.t(), String.t()}] + @type status() :: pos_integer() + + @spec request(atom(), String.t(), headers(), String.t(), keyword()) :: + {:ok, status(), headers} + | {:ok, status(), headers, map()} + | {:error, atom() | String.t()} + | no_return() + + @impl true + def request(method, url, headers, body, opts \\ []) do + check_adapter() + + opts = Keyword.put(opts, :body_as, :chunks) + + with {:ok, response} <- + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, opts) + ) do + if is_map(response.body) and method != :head do + {:ok, response.status, response.headers, response.body} + else + {:ok, response.status, response.headers} + end + else + {:error, error} -> {:error, error} + end + end + + @impl true + @spec stream_body(map()) :: + {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() + def stream_body(%{pid: pid, opts: opts, fin: true}) do + # if connection was reused, but in tesla were redirects, + # tesla returns new opened connection, which must be closed manually + if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid) + # if there were redirects we need to checkout old conn + conn = opts[:old_conn] || opts[:conn] + + if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) + + :done + end + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> + {:ok, body, Map.put(client, :fin, true)} + + {:nofin, part} -> + {:ok, part, client} + + {:error, error} -> + {:error, error} + end + end + + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do + adapter = check_adapter() + adapter.read_chunk(pid, stream, opts) + end + + @impl true + @spec close(map) :: :ok | no_return() + def close(%{pid: pid}) do + adapter = check_adapter() + adapter.close(pid) + end + + defp check_adapter do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter == Tesla.Adapter.Gun do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 8b713b8f4..4bbeb493c 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do - alias Pleroma.HTTP - @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified) @@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) - |> Keyword.merge(@default_hackney_options) - |> Keyword.merge(Keyword.get(opts, :http, [])) - |> HTTP.process_request_options() + client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, [])) req_headers = build_req_headers(conn.req_headers, opts) @@ -110,7 +104,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -156,11 +150,11 @@ defmodule Pleroma.ReverseProxy do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -210,7 +204,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index cc815430a..be01d541d 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do alias Pleroma.ThreadMute alias Pleroma.User - require Ecto.Query + import Ecto.Changeset + import Ecto.Query schema "thread_mutes" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) @@ -18,19 +19,44 @@ defmodule Pleroma.ThreadMute do def changeset(mute, params \\ %{}) do mute - |> Ecto.Changeset.cast(params, [:user_id, :context]) - |> Ecto.Changeset.foreign_key_constraint(:user_id) - |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index) + |> cast(params, [:user_id, :context]) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:user_id, name: :unique_index) end def query(user_id, context) do - {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) + user_binary_id = User.binary_id(user_id) ThreadMute - |> Ecto.Query.where(user_id: ^user_id) - |> Ecto.Query.where(context: ^context) + |> where(user_id: ^user_binary_id) + |> where(context: ^context) end + def muters_query(context) do + ThreadMute + |> join(:inner, [tm], u in assoc(tm, :user)) + |> where([tm], tm.context == ^context) + |> select([tm, u], u.ap_id) + end + + def muter_ap_ids(context, ap_ids \\ nil) + + # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.) + def muter_ap_ids(context, _ap_ids) when is_nil(context), do: [] + + def muter_ap_ids(context, ap_ids) do + context + |> muters_query() + |> maybe_filter_on_ap_id(ap_ids) + |> Repo.all() + end + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [tm, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def add_mute(user_id, context) do %ThreadMute{} |> changeset(%{user_id: user_id, context: context}) @@ -42,8 +68,8 @@ defmodule Pleroma.ThreadMute do |> Repo.delete_all() end - def check_muted(user_id, context) do + def exists?(user_id, context) do query(user_id, context) - |> Repo.all() + |> Repo.exists?() end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0fbac41ad..ff828aa17 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -150,22 +150,26 @@ defmodule Pleroma.User do {outgoing_relation, outgoing_relation_target}, {incoming_relation, incoming_relation_source} ]} <- @user_relationships_config do - # Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc. + # Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes, + # :notification_muter_mutes, :subscribee_subscriptions has_many(outgoing_relation, UserRelationship, foreign_key: :source_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc. + # Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes, + # :notification_mutee_mutes, :subscriber_subscriptions has_many(incoming_relation, UserRelationship, foreign_key: :target_id, where: [relationship_type: relationship_type] ) - # Definitions of `has_many :blocked_users`, `has_many :muted_users` etc. + # Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users, + # :notification_muted_users, :subscriber_users has_many(outgoing_relation_target, through: [outgoing_relation, :target]) - # Definitions of `has_many :blocker_users`, `has_many :muter_users` etc. + # Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users, + # :notification_muter_users, :subscribee_users has_many(incoming_relation_source, through: [incoming_relation, :source]) end @@ -185,7 +189,9 @@ defmodule Pleroma.User do for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <- @user_relationships_config do - # Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc. + # `def blocked_users_relation/2`, `def muted_users_relation/2`, + # `def reblog_muted_users_relation/2`, `def notification_muted_users/2`, + # `def subscriber_users/2` def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do target_users_query = assoc(user, unquote(outgoing_relation_target)) @@ -196,7 +202,8 @@ defmodule Pleroma.User do end end - # Definitions of `blocked_users/1`, `muted_users/1`, etc. + # `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`, + # `def notification_muted_users/2`, `def subscriber_users/2` def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -206,7 +213,8 @@ defmodule Pleroma.User do |> Repo.all() end - # Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc. + # `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`, + # `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2` def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do __MODULE__ |> apply(unquote(:"#{outgoing_relation_target}_relation"), [ @@ -218,6 +226,24 @@ defmodule Pleroma.User do end end + @doc """ + Dumps Flake Id to SQL-compatible format (16-byte UUID). + E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>> + """ + def binary_id(source_id) when is_binary(source_id) do + with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do + dumped_id + else + _ -> source_id + end + end + + def binary_id(source_ids) when is_list(source_ids) do + Enum.map(source_ids, &binary_id/1) + end + + def binary_id(%User{} = user), do: binary_id(user.id) + @doc "Returns status account" @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated @@ -279,16 +305,12 @@ defmodule Pleroma.User do end end - def profile_url(%User{source_data: %{"url" => url}}), do: url - def profile_url(%User{ap_id: ap_id}), do: ap_id - def profile_url(_), do: nil - def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" - @spec ap_following(User.t()) :: Sring.t() + @spec ap_following(User.t()) :: String.t() def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" @@ -745,7 +767,14 @@ defmodule Pleroma.User do def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) + get_follow_state(follower, following, following_relationship) + end + def get_follow_state( + %User{} = follower, + %User{} = following, + following_relationship + ) do case {following_relationship, following.local} do {nil, false} -> case Utils.fetch_latest_follow(follower, following) do @@ -1278,13 +1307,15 @@ defmodule Pleroma.User do end @doc """ - Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type. - E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` + Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type. + E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}` """ - @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} - def outgoing_relations_ap_ids(_, []), do: %{} + @spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())} + def outgoing_relationships_ap_ids(_user, []), do: %{} + + def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{} - def outgoing_relations_ap_ids(%User{} = user, relationship_types) + def outgoing_relationships_ap_ids(%User{} = user, relationship_types) when is_list(relationship_types) do db_result = user @@ -1303,6 +1334,30 @@ defmodule Pleroma.User do ) end + def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil) + + def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: [] + + def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: [] + + def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids) + when is_list(relationship_types) do + user + |> assoc(:incoming_relationships) + |> join(:inner, [user_rel], u in assoc(user_rel, :source)) + |> where([user_rel, u], user_rel.relationship_type in ^relationship_types) + |> maybe_filter_on_ap_id(ap_ids) + |> select([user_rel, u], u.ap_id) + |> distinct(true) + |> Repo.all() + end + + defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do + where(query, [user_rel, u], u.ap_id in ^ap_ids) + end + + defp maybe_filter_on_ap_id(query, _ap_ids), do: query + def deactivate_async(user, status \\ true) do BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end @@ -1713,8 +1768,12 @@ defmodule Pleroma.User do |> Repo.all() end + def muting_reblogs?(%User{} = user, %User{} = target) do + UserRelationship.reblog_mute_exists?(user, target) + end + def showing_reblogs?(%User{} = user, %User{} = target) do - not UserRelationship.reblog_mute_exists?(user, target) + not muting_reblogs?(user, target) end @doc """ diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 393947942..18a5eec72 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias Pleroma.FollowingRelationship alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserRelationship @@ -21,19 +22,26 @@ defmodule Pleroma.UserRelationship do end for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do - # Definitions of `create_block/2`, `create_mute/2` etc. + # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`, + # `def create_notification_mute/2`, `def create_inverse_subscription/2` def unquote(:"create_#{relationship_type}")(source, target), do: create(unquote(relationship_type), source, target) - # Definitions of `delete_block/2`, `delete_mute/2` etc. + # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`, + # `def delete_notification_mute/2`, `def delete_inverse_subscription/2` def unquote(:"delete_#{relationship_type}")(source, target), do: delete(unquote(relationship_type), source, target) - # Definitions of `block_exists?/2`, `mute_exists?/2` etc. + # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`, + # `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2` def unquote(:"#{relationship_type}_exists?")(source, target), do: exists?(unquote(relationship_type), source, target) end + def user_relationship_types, do: Keyword.keys(user_relationship_mappings()) + + def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__() + def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do user_relationship |> cast(params, [:relationship_type, :source_id, :target_id]) @@ -72,6 +80,73 @@ defmodule Pleroma.UserRelationship do end end + def dictionary( + source_users, + target_users, + source_to_target_rel_types \\ nil, + target_to_source_rel_types \\ nil + ) + when is_list(source_users) and is_list(target_users) do + source_user_ids = User.binary_id(source_users) + target_user_ids = User.binary_id(target_users) + + get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end + + source_to_target_rel_types = + Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + target_to_source_rel_types = + Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1)) + + __MODULE__ + |> where( + fragment( + "(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \ + (source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))", + ^source_user_ids, + ^target_user_ids, + ^source_to_target_rel_types, + ^target_user_ids, + ^source_user_ids, + ^target_to_source_rel_types + ) + ) + |> select([ur], [ur.relationship_type, ur.source_id, ur.target_id]) + |> Repo.all() + end + + def exists?(dictionary, rel_type, source, target, func) do + cond do + is_nil(source) or is_nil(target) -> + false + + dictionary -> + [rel_type, source.id, target.id] in dictionary + + true -> + func.(source, target) + end + end + + @doc ":relationships option for StatusView / AccountView / NotificationView" + def view_relationships_option(nil = _reading_user, _actors) do + %{user_relationships: [], following_relationships: []} + end + + def view_relationships_option(%User{} = reading_user, actors) do + user_relationships = + UserRelationship.dictionary( + [reading_user], + actors, + [:block, :mute, :notification_mute, :reblog_mute], + [:block, :inverse_subscription] + ) + + following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) + + %{user_relationships: user_relationships, following_relationships: following_relationships} + end + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do changeset |> validate_change(:target_id, fn _, target_id -> diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 974231925..19286fd01 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -125,6 +125,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop + @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} + def persist(object, meta) do + with local <- Keyword.fetch!(meta, :local), + {recipients, _, _} <- get_recipients(object), + {:ok, activity} <- + Repo.insert(%Activity{ + data: object, + local: local, + recipients: recipients, + actor: object["actor"] + }) do + {:ok, activity, meta} + end + end + @spec insert(map(), boolean(), boolean(), boolean()) :: {:ok, Activity.t()} | {:error, any()} def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), @@ -1239,17 +1254,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp fetch_activities_query_ap_ids_ops(opts) do source_user = opts["muting_user"] - ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: [] + ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] - ap_id_relations = - ap_id_relations ++ + ap_id_relationships = + ap_id_relationships ++ if opts["blocking_user"] && opts["blocking_user"] == source_user do [:block] else [] end - preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations) + preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships) restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts) restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts) @@ -1379,6 +1394,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec get_actor_url(any()) :: binary() | nil + defp get_actor_url(url) when is_binary(url), do: url + defp get_actor_url(%{"href" => href}) when is_binary(href), do: href + + defp get_actor_url(url) when is_list(url) do + url + |> List.first() + |> get_actor_url() + end + + defp get_actor_url(_url), do: nil + defp object_to_user_data(data) do avatar = data["icon"]["url"] && @@ -1408,6 +1435,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do user_data = %{ ap_id: data["id"], + uri: get_actor_url(data["url"]), ap_enabled: true, source_data: data, banner: banner, diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex new file mode 100644 index 000000000..429a510b8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -0,0 +1,43 @@ +defmodule Pleroma.Web.ActivityPub.Builder do + @moduledoc """ + This module builds the objects. Meant to be used for creating local objects. + + This module encodes our addressing policies and general shape of our objects. + """ + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Visibility + + @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()} + def like(actor, object) do + object_actor = User.get_cached_by_ap_id(object.data["actor"]) + + # Address the actor of the object, and our actor's follower collection if the post is public. + to = + if Visibility.is_public?(object) do + [actor.follower_address, object.data["actor"]] + else + [object.data["actor"]] + end + + # CC everyone who's been addressed in the object, except ourself and the object actor's + # follower collection + cc = + (object.data["to"] ++ (object.data["cc"] || [])) + |> List.delete(actor.ap_id) + |> List.delete(object_actor.follower_address) + + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "actor" => actor.ap_id, + "type" => "Like", + "object" => object.data["id"], + "to" => to, + "cc" => cc, + "context" => object.data["context"] + }, []} + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index b3547ecd4..0270b96ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index d9a0acfd3..dfab105a3 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ - pool: :media, - recv_timeout: 10_000 + @options [ + pool: :media ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.put(@options, :recv_timeout, 10_000) + else + @options + end + url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: opts) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f67f48ab6..fc3475048 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex new file mode 100644 index 000000000..dc4bce059 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidator do + @moduledoc """ + This module is responsible for validating an object (which can be an activity) + and checking if it is both well formed and also compatible with our view of + the system. + """ + + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + + @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} + def validate(object, meta) + + def validate(%{"type" => "Like"} = object, meta) do + with {:ok, object} <- + object |> LikeValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object |> Map.from_struct()) + {:ok, object, meta} + end + end + + def stringify_keys(object) do + object + |> Map.new(fn {key, val} -> {to_string(key), val} end) + end + + def fetch_actor_and_object(object) do + User.get_or_fetch_by_ap_id(object["actor"]) + Object.normalize(object["object"]) + :ok + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex new file mode 100644 index 000000000..b479c3918 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do + import Ecto.Changeset + + alias Pleroma.Object + alias Pleroma.User + + def validate_actor_presence(cng, field_name \\ :actor) do + cng + |> validate_change(field_name, fn field_name, actor -> + if User.get_cached_by_ap_id(actor) do + [] + else + [{field_name, "can't find user"}] + end + end) + end + + def validate_object_presence(cng, field_name \\ :object) do + cng + |> validate_change(field_name, fn field_name, object -> + if Object.get_cached_by_ap_id(object) do + [] + else + [{field_name, "can't find object"}] + end + end) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex new file mode 100644 index 000000000..926804ce7 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/create_validator.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:actor, Types.ObjectID) + field(:type, :string) + field(:to, {:array, :string}) + field(:cc, {:array, :string}) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + + embeds_one(:object, NoteValidator) + end + + def cast_data(data) do + cast(%__MODULE__{}, data, __schema__(:fields)) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex new file mode 100644 index 000000000..49546ceaa --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.Web.ActivityPub.Utils + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:type, :string) + field(:object, Types.ObjectID) + field(:actor, Types.ObjectID) + field(:context, :string) + field(:to, {:array, :string}) + field(:cc, {:array, :string}) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, [:id, :type, :object, :actor, :context, :to, :cc]) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Like"]) + |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) + |> validate_actor_presence() + |> validate_object_presence() + |> validate_existing_like() + end + + def validate_existing_like(%{changes: %{actor: actor, object: object}} = cng) do + if Utils.get_existing_like(actor, %{data: %{"id" => object}}) do + cng + |> add_error(:actor, "already liked this object") + |> add_error(:object, "already liked by this actor") + else + cng + end + end + + def validate_existing_like(cng), do: cng +end diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex new file mode 100644 index 000000000..c95b622e4 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:to, {:array, :string}, default: []) + field(:cc, {:array, :string}, default: []) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + field(:actor, Types.ObjectID) + field(:attributedTo, Types.ObjectID) + field(:summary, :string) + field(:published, Types.DateTime) + # TODO: Write type + field(:emoji, :map, default: %{}) + field(:sensitive, :boolean, default: false) + # TODO: Write type + field(:attachment, {:array, :map}, default: []) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inRepyTo, :string) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + + # see if needed + field(:conversation, :string) + field(:context_id, :string) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Note"]) + |> validate_required([:id, :actor, :to, :cc, :type, :content, :context]) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex new file mode 100644 index 000000000..4f412fcde --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex @@ -0,0 +1,34 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do + @moduledoc """ + The AP standard defines the date fields in AP as xsd:DateTime. Elixir's + DateTime can't parse this, but it can parse the related iso8601. This + module punches the date until it looks like iso8601 and normalizes to + it. + + DateTimes without a timezone offset are treated as UTC. + + Reference: https://www.w3.org/TR/activitystreams-vocabulary/#dfn-published + """ + use Ecto.Type + + def type, do: :string + + def cast(datetime) when is_binary(datetime) do + with {:ok, datetime, _} <- DateTime.from_iso8601(datetime) do + {:ok, DateTime.to_iso8601(datetime)} + else + {:error, :missing_offset} -> cast("#{datetime}Z") + _e -> :error + end + end + + def cast(_), do: :error + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex new file mode 100644 index 000000000..ee10be0b0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do + use Ecto.Type + + def type, do: :string + + def cast(object) when is_binary(object) do + # Host has to be present and scheme has to be an http scheme (for now) + case URI.parse(object) do + %URI{host: nil} -> + :error + + %URI{scheme: scheme} when scheme in ["https", "http"] -> + {:ok, object} + + _ -> + :error + end + end + + def cast(%{"id" => object}), do: cast(object) + + def cast(_) do + :error + end + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex new file mode 100644 index 000000000..7ccee54c9 --- /dev/null +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Pipeline do + alias Pleroma.Activity + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.MRF + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.SideEffects + alias Pleroma.Web.Federator + + @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()} + def common_pipeline(object, meta) do + with {_, {:ok, validated_object, meta}} <- + {:validate_object, ObjectValidator.validate(object, meta)}, + {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:persist_object, ActivityPub.persist(mrfd_object, meta)}, + {_, {:ok, %Activity{} = activity, meta}} <- + {:execute_side_effects, SideEffects.handle(activity, meta)}, + {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do + {:ok, activity, meta} + else + {:mrf_object, {:reject, _}} -> {:ok, nil, meta} + e -> {:error, e} + end + end + + defp maybe_federate(activity, meta) do + with {:ok, local} <- Keyword.fetch(meta, :local) do + if local do + Federator.publish(activity) + {:ok, :federated} + else + {:ok, :not_federated} + end + else + _e -> {:error, :badarg} + end + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex new file mode 100644 index 000000000..666a4e310 --- /dev/null +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -0,0 +1,28 @@ +defmodule Pleroma.Web.ActivityPub.SideEffects do + @moduledoc """ + This module looks at an inserted object and executes the side effects that it + implies. For example, a `Like` activity will increase the like count on the + liked object, a `Follow` activity will add the user to the follower + collection, and so on. + """ + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Utils + + def handle(object, meta \\ []) + + # Tasks this handles: + # - Add like to object + # - Set up notification + def handle(%{data: %{"type" => "Like"}} = object, meta) do + liked_object = Object.get_by_ap_id(object.data["object"]) + Utils.add_like_to_object(object, liked_object) + Notification.create_notifications(object) + {:ok, object, meta} + end + + # Nothing to do + def handle(object, meta) do + {:ok, object, meta} + end +end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..0a8ad62ad 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -13,6 +13,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.ObjectValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator @@ -229,7 +232,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do + def fix_url(%{"type" => object_type, "url" => url} = object) + when object_type in ["Video", "Audio"] and is_list(url) do first_element = Enum.at(url, 0) link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) @@ -398,7 +402,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do actor = Containment.get_actor(data) data = @@ -608,17 +612,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> handle_incoming(options) end - def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, - _options - ) do - with actor <- Containment.get_actor(data), - {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id), - {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do + def handle_incoming(%{"type" => "Like"} = data, _options) do + with {_, {:ok, cast_data_sym}} <- + {:casting_data, + data |> LikeValidator.cast_data() |> Ecto.Changeset.apply_action(:insert)}, + cast_data = ObjectValidator.stringify_keys(Map.from_struct(cast_data_sym)), + :ok <- ObjectValidator.fetch_actor_and_object(cast_data), + {_, {:ok, cast_data}} <- {:ensure_context_presence, ensure_context_presence(cast_data)}, + {_, {:ok, cast_data}} <- + {:ensure_recipients_presence, ensure_recipients_presence(cast_data)}, + {_, {:ok, activity, _meta}} <- + {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do {:ok, activity} else - _e -> :error + e -> {:error, e} end end @@ -1108,13 +1115,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def add_mention_tags(object) do - mentions = - object - |> Utils.get_notified_from_object() - |> Enum.map(&build_mention_tag/1) + {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object) + potential_receivers = enabled_receivers ++ disabled_receivers + mentions = Enum.map(potential_receivers, &build_mention_tag/1) tags = object["tag"] || [] - Map.put(object, "tag", tags ++ mentions) end @@ -1244,4 +1249,45 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_user_url(data), do: data def maybe_fix_user_object(data), do: maybe_fix_user_url(data) + + defp ensure_context_presence(%{"context" => context} = data) when is_binary(context), + do: {:ok, data} + + defp ensure_context_presence(%{"object" => object} = data) when is_binary(object) do + with %{data: %{"context" => context}} when is_binary(context) <- Object.normalize(object) do + {:ok, Map.put(data, "context", context)} + else + _ -> + {:error, :no_context} + end + end + + defp ensure_context_presence(_) do + {:error, :no_context} + end + + defp ensure_recipients_presence(%{"to" => [_ | _], "cc" => [_ | _]} = data), + do: {:ok, data} + + defp ensure_recipients_presence(%{"object" => object} = data) do + case Object.normalize(object) do + %{data: %{"actor" => actor}} -> + data = + data + |> Map.put("to", [actor]) + |> Map.put("cc", data["cc"] || []) + + {:ok, data} + + nil -> + {:error, :no_object} + + _ -> + {:error, :no_actor} + end + end + + defp ensure_recipients_presence(_) do + {:error, :no_object} + end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c65bbed67..2d685ecc0 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -795,102 +795,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do ActivityPub.fetch_activities([], params, :offset) end - def parse_report_group(activity) do - reports = get_reports_by_status_id(activity["id"]) - max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) - actors = Enum.map(reports, & &1.user_actor) - [%{data: %{"object" => [account_id | _]}} | _] = reports - - account = - AccountView.render("show.json", %{ - user: User.get_by_ap_id(account_id) - }) - - status = get_status_data(activity) - - %{ - date: max_date.data["published"], - account: account, - status: status, - actors: Enum.uniq(actors), - reports: reports - } - end - - defp get_status_data(status) do - case status["deleted"] do - true -> - %{ - "id" => status["id"], - "deleted" => true - } - - _ -> - Activity.get_by_ap_id(status["id"]) - end - end - - def get_reports_by_status_id(ap_id) do - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - where: fragment("(?)->'object' @> ?", a.data, ^[%{id: ap_id}]), - or_where: fragment("(?)->'object' @> ?", a.data, ^[ap_id]) - ) - |> Activity.with_preloaded_user_actor() - |> Repo.all() - end - - @spec get_reports_grouped_by_status([String.t()]) :: %{ - required(:groups) => [ - %{ - required(:date) => String.t(), - required(:account) => %{}, - required(:status) => %{}, - required(:actors) => [%User{}], - required(:reports) => [%Activity{}] - } - ] - } - def get_reports_grouped_by_status(activity_ids) do - parsed_groups = - activity_ids - |> Enum.map(fn id -> - id - |> build_flag_object() - |> parse_report_group() - end) - - %{ - groups: parsed_groups - } - end - - @spec get_reported_activities() :: [ - %{ - required(:activity) => String.t(), - required(:date) => String.t() - } - ] - def get_reported_activities do - reported_activities_query = - from(a in Activity, - where: fragment("(?)->>'type' = 'Flag'", a.data), - select: %{ - activity: fragment("jsonb_array_elements((? #- '{object,0}')->'object')", a.data) - }, - group_by: fragment("activity") - ) - - from(a in subquery(reported_activities_query), - distinct: true, - select: %{ - id: fragment("COALESCE(?->>'id'::text, ? #>> '{}')", a.activity, a.activity) - } - ) - |> Repo.all() - |> Enum.map(& &1.id) - end - def update_report_state(%Activity{} = activity, state) when state in @strip_status_report_states do {:ok, stripped_activity} = strip_report_status_data(activity) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0368df1e9..ca5439920 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -715,14 +715,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{reports: reports}) end - def list_grouped_reports(conn, _params) do - statuses = Utils.get_reported_activities() - - conn - |> put_view(ReportView) - |> render("index_grouped.json", Utils.get_reports_grouped_by_status(statuses)) - end - def report_show(conn, %{"id" => id}) do with %Activity{} = report <- Activity.get_by_id(id) do conn diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index fc8733ce8..ca0bcebc7 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view - alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.AdminAPI.Report @@ -44,32 +44,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do } end - def render("index_grouped.json", %{groups: groups}) do - reports = - Enum.map(groups, fn group -> - status = - case group.status do - %Activity{} = activity -> StatusView.render("show.json", %{activity: activity}) - _ -> group.status - end - - %{ - date: group[:date], - account: group[:account], - status: Map.put_new(status, "deleted", false), - actors: Enum.map(group[:actors], &merge_account_views/1), - reports: - group[:reports] - |> Enum.map(&Report.extract_report_info(&1)) - |> Enum.map(&render(__MODULE__, "show.json", &1)) - } - end) - - %{ - reports: reports - } - end - def render("index_notes.json", %{notes: notes}) when is_list(notes) do Enum.map(notes, &render(__MODULE__, "show_note.json", &1)) end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 091011c6b..636cf3301 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -19,6 +21,7 @@ defmodule Pleroma.Web.CommonAPI do import Pleroma.Web.CommonAPI.Utils require Pleroma.Constants + require Logger def follow(follower, followed) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) @@ -109,18 +112,51 @@ defmodule Pleroma.Web.CommonAPI do end end - def favorite(id_or_ap_id, user) do - with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)}, - object <- Object.normalize(activity), - like_activity <- Utils.get_existing_like(user.ap_id, object) do - if like_activity do - {:ok, like_activity, object} - else - ActivityPub.like(user, object) - end + @spec favorite(User.t(), binary()) :: {:ok, Activity.t() | :already_liked} | {:error, any()} + def favorite(%User{} = user, id) do + case favorite_helper(user, id) do + {:ok, _} = res -> + res + + {:error, :not_found} = res -> + res + + {:error, e} -> + Logger.error("Could not favorite #{id}. Error: #{inspect(e, pretty: true)}") + {:error, dgettext("errors", "Could not favorite")} + end + end + + def favorite_helper(user, id) do + with {_, %Activity{object: object}} <- {:find_object, Activity.get_by_id_with_object(id)}, + {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, + {_, {:ok, %Activity{} = activity, _meta}} <- + {:common_pipeline, + Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do + {:ok, activity} else - {:find_activity, _} -> {:error, :not_found} - _ -> {:error, dgettext("errors", "Could not favorite")} + {:find_object, _} -> + {:error, :not_found} + + {:common_pipeline, + { + :error, + { + :validate_object, + { + :error, + changeset + } + } + }} = e -> + if {:object, {"already liked by this actor", []}} in changeset.errors do + {:ok, :already_liked} + else + {:error, e} + end + + e -> + {:error, e} end end @@ -358,7 +394,7 @@ defmodule Pleroma.Web.CommonAPI do def thread_muted?(%{id: nil} = _user, _activity), do: false def thread_muted?(user, activity) do - ThreadMute.check_muted(user.id, activity.data["context"]) != [] + ThreadMute.exists?(user.id, activity.data["context"]) end def report(user, %{"account_id" => account_id} = data) do diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index ad293cda9..b49523ec3 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -34,7 +34,12 @@ defmodule Pleroma.Web.ControllerHelper do defp param_to_integer(_, default), do: default - def add_link_headers(conn, activities, extra_params \\ %{}) do + def add_link_headers(conn, activities, extra_params \\ %{}) + + def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params), + do: conn + + def add_link_headers(conn, activities, extra_params) do case List.last(activities) do %{id: max_id} -> params = diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index fb80dd348..21bc3d5a5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -62,11 +62,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do when action not in [:create, :show, :statuses] ) - @relations [:follow, :unfollow] + @relationship_actions [:follow, :unfollow] @needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a - plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations) - plug(RateLimiter, [name: :relations_actions] when action in @relations) + plug( + RateLimiter, + [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions + ) + + plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions) plug(RateLimiter, [name: :app_account_creation] when action == :create) plug(:assign_account_by_id when action in @needs_account) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 37afe6949..ec8f0d8a0 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -207,9 +207,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "POST /api/v1/statuses/:id/favourite" - def favourite(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do - with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do + def favourite(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) do + with {:ok, _fav} <- CommonAPI.favorite(user, activity_id), + %Activity{} = activity <- Activity.get_by_id(activity_id) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2bf711386..99e62f580 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -5,12 +5,30 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.FollowingRelationship alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy def render("index.json", %{users: users} = opts) do + reading_user = opts[:for] + + relationships_opt = + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(reading_user) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(reading_user, users) + end + + opts = Map.put(opts, :relationships, relationships_opt) + users |> render_many(AccountView, "show.json", opts) |> Enum.filter(&Enum.any?/1) @@ -27,7 +45,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: User.profile_url(user) + url: user.uri || user.ap_id } end @@ -35,27 +53,107 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do - follow_state = User.get_follow_state(user, target) + def render( + "relationship.json", + %{user: %User{} = reading_user, target: %User{} = target} = opts + ) do + user_relationships = get_in(opts, [:relationships, :user_relationships]) + following_relationships = get_in(opts, [:relationships, :following_relationships]) + + follow_state = + if following_relationships do + user_to_target_following_relation = + FollowingRelationship.find(following_relationships, reading_user, target) + User.get_follow_state(reading_user, target, user_to_target_following_relation) + else + User.get_follow_state(reading_user, target) + end + + followed_by = + if following_relationships do + case FollowingRelationship.find(following_relationships, target, reading_user) do + %{state: "accept"} -> true + _ -> false + end + else + User.following?(target, reading_user) + end + + # NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags %{ id: to_string(target.id), following: follow_state == "accept", - followed_by: User.following?(target, user), - blocking: User.blocks_user?(user, target), - blocked_by: User.blocks_user?(target, user), - muting: User.mutes?(user, target), - muting_notifications: User.muted_notifications?(user, target), - subscribing: User.subscribed_to?(user, target), + followed_by: followed_by, + blocking: + UserRelationship.exists?( + user_relationships, + :block, + reading_user, + target, + &User.blocks_user?(&1, &2) + ), + blocked_by: + UserRelationship.exists?( + user_relationships, + :block, + target, + reading_user, + &User.blocks_user?(&1, &2) + ), + muting: + UserRelationship.exists?( + user_relationships, + :mute, + reading_user, + target, + &User.mutes?(&1, &2) + ), + muting_notifications: + UserRelationship.exists?( + user_relationships, + :notification_mute, + reading_user, + target, + &User.muted_notifications?(&1, &2) + ), + subscribing: + UserRelationship.exists?( + user_relationships, + :inverse_subscription, + target, + reading_user, + &User.subscribed_to?(&2, &1) + ), requested: follow_state == "pending", - domain_blocking: User.blocks_domain?(user, target), - showing_reblogs: User.showing_reblogs?(user, target), + domain_blocking: User.blocks_domain?(reading_user, target), + showing_reblogs: + not UserRelationship.exists?( + user_relationships, + :reblog_mute, + reading_user, + target, + &User.muting_reblogs?(&1, &2) + ), endorsed: false } end - def render("relationships.json", %{user: user, targets: targets}) do - render_many(targets, AccountView, "relationship.json", user: user, as: :target) + def render("relationships.json", %{user: user, targets: targets} = opts) do + relationships_opt = + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(user) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + UserRelationship.view_relationships_option(user, targets) + end + + render_opts = %{as: :target, user: user, relationships: relationships_opt} + render_many(targets, AccountView, "relationship.json", render_opts) end defp do_render("show.json", %{user: user} = opts) do @@ -93,7 +191,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - relationship = render("relationship.json", %{user: opts[:for], target: user}) + relationship = + render("relationship.json", %{ + user: opts[:for], + target: user, + relationships: opts[:relationships] + }) %{ id: to_string(user.id), @@ -106,7 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do following_count: following_count, statuses_count: user.note_count, note: user.bio || "", - url: User.profile_url(user), + url: user.uri || user.ap_id, avatar: image, avatar_static: image, header: header, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 33145c484..ae87d4701 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -8,24 +8,86 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - def render("index.json", %{notifications: notifications, for: user}) do - safe_render_many(notifications, NotificationView, "show.json", %{for: user}) + def render("index.json", %{notifications: notifications, for: reading_user} = opts) do + activities = Enum.map(notifications, & &1.activity) + + parent_activities = + activities + |> Enum.filter( + &(Activity.mastodon_notification_type(&1) in [ + "favourite", + "reblog", + "pleroma:emoji_reaction" + ]) + ) + |> Enum.map(& &1.data["object"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Pleroma.Repo.all() + + relationships_opt = + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(reading_user) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + move_activities_targets = + activities + |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"])) + + actors = + activities + |> Enum.map(fn a -> User.get_cached_by_ap_id(a.data["actor"]) end) + |> Enum.filter(& &1) + |> Kernel.++(move_activities_targets) + + UserRelationship.view_relationships_option(reading_user, actors) + end + + opts = %{ + for: reading_user, + parent_activities: parent_activities, + relationships: relationships_opt + } + + safe_render_many(notifications, NotificationView, "show.json", opts) end - def render("show.json", %{ - notification: %Notification{activity: activity} = notification, - for: user - }) do + def render( + "show.json", + %{ + notification: %Notification{activity: activity} = notification, + for: reading_user + } = opts + ) do actor = User.get_cached_by_ap_id(activity.data["actor"]) - parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + + parent_activity_fn = fn -> + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + else + Activity.get_create_by_object_ap_id(activity.data["object"]) + end + end + mastodon_type = Activity.mastodon_notification_type(activity) - with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do + with %{id: _} = account <- + AccountView.render("show.json", %{ + user: actor, + for: reading_user, + relationships: opts[:relationships] + }) do response = %{ id: to_string(notification.id), type: mastodon_type, @@ -36,24 +98,28 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do } } + render_opts = %{relationships: opts[:relationships]} + case mastodon_type do "mention" -> - put_status(response, activity, user) + put_status(response, activity, reading_user, render_opts) "favourite" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "reblog" -> - put_status(response, parent_activity, user) + put_status(response, parent_activity_fn.(), reading_user, render_opts) "move" -> - put_target(response, activity, user) + put_target(response, activity, reading_user, render_opts) "follow" -> response "pleroma:emoji_reaction" -> - put_status(response, parent_activity, user) |> put_emoji(activity) + response + |> put_status(parent_activity_fn.(), reading_user, render_opts) + |> put_emoji(activity) _ -> nil @@ -64,16 +130,21 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do end defp put_emoji(response, activity) do - response - |> Map.put(:emoji, activity.data["content"]) + Map.put(response, :emoji, activity.data["content"]) end - defp put_status(response, activity, user) do - Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user})) + defp put_status(response, activity, reading_user, opts) do + status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user}) + status_render = StatusView.render("show.json", status_render_opts) + + Map.put(response, :status, status_render) end - defp put_target(response, activity, user) do - target = User.get_cached_by_ap_id(activity.data["target"]) - Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user})) + defp put_target(response, activity, reading_user, opts) do + target_user = User.get_cached_by_ap_id(activity.data["target"]) + target_render_opts = Map.merge(opts, %{user: target_user, for: reading_user}) + target_render = AccountView.render("show.json", target_render_opts) + + Map.put(response, :target, target_render) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f7469cdff..cea76e735 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -71,10 +72,43 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render("index.json", opts) do - replied_to_activities = get_replied_to_activities(opts.activities) - opts = Map.put(opts, :replied_to_activities, replied_to_activities) + reading_user = opts[:for] - safe_render_many(opts.activities, StatusView, "show.json", opts) + # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list + activities = Enum.filter(opts.activities, & &1) + replied_to_activities = get_replied_to_activities(activities) + + parent_activities = + activities + |> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"])) + |> Enum.map(&Object.normalize(&1).data["id"]) + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object(:left) + |> Activity.with_preloaded_bookmark(reading_user) + |> Activity.with_set_thread_muted_field(reading_user) + |> Repo.all() + + relationships_opt = + cond do + Map.has_key?(opts, :relationships) -> + opts[:relationships] + + is_nil(reading_user) -> + UserRelationship.view_relationships_option(nil, []) + + true -> + actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) + + UserRelationship.view_relationships_option(reading_user, actors) + end + + opts = + opts + |> Map.put(:replied_to_activities, replied_to_activities) + |> Map.put(:parent_activities, parent_activities) + |> Map.put(:relationships, relationships_opt) + + safe_render_many(activities, StatusView, "show.json", opts) end def render( @@ -85,17 +119,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do created_at = Utils.to_masto_date(activity.data["published"]) activity_object = Object.normalize(activity) - reblogged_activity = - Activity.create_by_object_ap_id(activity_object.data["id"]) - |> Activity.with_preloaded_bookmark(opts[:for]) - |> Activity.with_set_thread_muted_field(opts[:for]) - |> Repo.one() + reblogged_parent_activity = + if opts[:parent_activities] do + Activity.Queries.find_by_object_ap_id( + opts[:parent_activities], + activity_object.data["id"] + ) + else + Activity.create_by_object_ap_id(activity_object.data["id"]) + |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) + |> Repo.one() + end - reblogged = render("show.json", Map.put(opts, :activity, reblogged_activity)) + reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity) + reblogged = render("show.json", reblog_rendering_opts) favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) - bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil + bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil mentions = activity.recipients @@ -107,7 +149,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: activity_object.data["id"], url: activity_object.data["id"], - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + relationships: opts[:relationships] + }), in_reply_to_id: nil, in_reply_to_account_id: nil, reblog: reblogged, @@ -116,7 +163,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogs_count: 0, replies_count: 0, favourites_count: 0, - reblogged: reblogged?(reblogged_activity, opts[:for]), + reblogged: reblogged?(reblogged_parent_activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), muted: false, @@ -183,9 +230,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end thread_muted? = - case activity.thread_muted? do - thread_muted? when is_boolean(thread_muted?) -> thread_muted? - nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false + cond do + is_nil(opts[:for]) -> false + is_boolean(activity.thread_muted?) -> activity.thread_muted? + true -> CommonAPI.thread_muted?(opts[:for], activity) end attachment_data = object.data["attachment"] || [] @@ -253,11 +301,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> [] end + muted = + thread_muted? || + UserRelationship.exists?( + get_in(opts, [:relationships, :user_relationships]), + :mute, + opts[:for], + user, + fn for_user, user -> User.mutes?(for_user, user) end + ) + %{ id: to_string(activity.id), uri: object.data["id"], url: url, - account: AccountView.render("show.json", %{user: user, for: opts[:for]}), + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for], + relationships: opts[:relationships] + }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, @@ -270,7 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: thread_muted? || User.mutes?(opts[:for], user), + muted: muted, pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary, @@ -421,7 +484,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render_content(%{data: %{"type" => object_type}} = object) - when object_type in ["Video", "Event"] do + when object_type in ["Video", "Event", "Audio"] do with name when not is_nil(name) and name != "" <- object.data["name"] do "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}" else diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index c9aac27dc..a9f70c43e 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -6,7 +6,12 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML def build_tags(params) do - Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc -> + providers = [ + Pleroma.Web.Metadata.Providers.RestrictIndexing + | Pleroma.Config.get([__MODULE__, :providers], []) + ] + + Enum.reduce(providers, "", fn parser, acc -> rendered_html = params |> parser.build_tags() diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 21446ac77..68c871e71 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do property: "og:title", content: Utils.user_name_string(user) ], []}, - {:meta, [property: "og:url", content: User.profile_url(user)], []}, + {:meta, [property: "og:url", content: user.uri || user.ap_id], []}, {:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:type", content: "website"], []}, {:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []}, diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex new file mode 100644 index 000000000..f15607896 --- /dev/null +++ b/lib/pleroma/web/metadata/restrict_indexing.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do + @behaviour Pleroma.Web.Metadata.Providers.Provider + + @moduledoc """ + Restricts indexing of remote users. + """ + + @impl true + def build_tags(%{user: %{local: false}}) do + [ + {:meta, + [ + name: "robots", + content: "noindex, noarchive" + ], []} + ] + end + + @impl true + def build_tags(%{user: %{local: true}}), do: [] +end diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index e97c398dc..8e2b51508 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] if Pleroma.Config.get(:env) == :test do @@ -25,8 +23,18 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: @hackney_options), + Pleroma.HTTP.get(url, [], adapter: opts), {:ok, html_tree} <- Floki.parse_document(html), data <- Floki.attribute(html_tree, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 0779065ee..40980def8 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,11 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, - recv_timeout: 2_000, - max_body: 2_000_000, - with_body: true + max_body: 2_000_000 ] defp parsers do @@ -77,8 +75,18 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp parse_url(url) do + opts = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do + Keyword.merge(@options, + recv_timeout: 2_000, + with_body: true + ) + else + @options + end + try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) html |> parse_html() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a22f744c1..5a0902739 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -186,7 +186,6 @@ defmodule Pleroma.Web.Router do patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email) get("/reports", AdminAPIController, :list_reports) - get("/grouped_reports", AdminAPIController, :list_grouped_reports) get("/reports/:id", AdminAPIController, :report_show) patch("/reports", AdminAPIController, :reports_update) post("/reports/:id/notes", AdminAPIController, :report_notes_create) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index 29f992a67..abfed21c8 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.Streamer.Worker do defp should_send?(%User{} = user, %Activity{} = item) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = - User.outgoing_relations_ap_ids(user, [:block, :mute, :reblog_mute]) + User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids) recipients = MapSet.new(item.recipients) diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex index c7789f9ac..2a7582d45 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex @@ -1,5 +1,5 @@ <div class="p-author h-card"> - <a class="u-url" rel="author noopener" href="<%= User.profile_url(@user) %>"> + <a class="u-url" rel="author noopener" href="<%= (@user.uri || @user.ap_id) %>"> <div class="avatar"> <img src="<%= User.avatar_url(@user) |> MediaProxy.url %>" width="48" height="48" alt=""> </div> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 94063c92d..e7d2aecad 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -8,7 +8,7 @@ <button type="submit" class="collapse">Remote follow</button> </form> <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> | - <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %> + <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %> </h3> <p><%= raw @user.bio %></p> </header> diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 43a81c75d..7ffd0e51b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -173,7 +173,8 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body, status: status}} when status in 200..299 <- + HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} @@ -205,7 +206,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) |