diff options
| author | Alex Gleason <alex@alexgleason.me> | 2021-05-29 11:35:57 -0500 | 
|---|---|---|
| committer | Alex Gleason <alex@alexgleason.me> | 2021-05-29 11:35:57 -0500 | 
| commit | e2ba852bfe8e1fa0cf25463e57185ba9cfabf429 (patch) | |
| tree | 881c3fed55e106a4cc0ea60bcd56e0653fb0fb9f /lib | |
| parent | 0ada3fe823a3c2e6c5835431bdacfbdb8b3d02a7 (diff) | |
| parent | 8871ca5aa35e9533e57b4a15420687869378a981 (diff) | |
| download | pleroma-e2ba852bfe8e1fa0cf25463e57185ba9cfabf429.tar.gz pleroma-e2ba852bfe8e1fa0cf25463e57185ba9cfabf429.zip  | |
Merge remote-tracking branch 'pleroma/develop' into cycles-gun
Diffstat (limited to 'lib')
114 files changed, 2849 insertions, 979 deletions
diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 1962154b9..22502a522 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Config do        {opts, _} =          OptionParser.parse!(options, -          strict: [env: :string, delete: :boolean], +          strict: [env: :string, delete: :boolean, path: :string],            aliases: [d: :delete]          ) @@ -259,18 +259,43 @@ defmodule Mix.Tasks.Pleroma.Config do    defp migrate_from_db(opts) do      env = opts[:env] || Pleroma.Config.get(:env) +    filename = "#{env}.exported_from_db.secret.exs" +      config_path = -      if Pleroma.Config.get(:release) do -        :config_path -        |> Pleroma.Config.get() -        |> Path.dirname() -      else -        "config" +      cond do +        opts[:path] -> +          opts[:path] + +        Pleroma.Config.get(:release) -> +          :config_path +          |> Pleroma.Config.get() +          |> Path.dirname() + +        true -> +          "config"        end -      |> Path.join("#{env}.exported_from_db.secret.exs") +      |> Path.join(filename) -    file = File.open!(config_path, [:write, :utf8]) +    with {:ok, file} <- File.open(config_path, [:write, :utf8]) do +      write_config(file, config_path, opts) +      shell_info("Database configuration settings have been exported to #{config_path}") +    else +      _ -> +        shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}") +        tmp_config_path = Path.join(System.tmp_dir!(), filename) +        file = File.open!(tmp_config_path) + +        shell_info( +          "Saving database configuration settings to #{tmp_config_path}. Copy it to the #{ +            Path.dirname(config_path) +          } manually." +        ) +        write_config(file, tmp_config_path, opts) +    end +  end + +  defp write_config(file, path, opts) do      IO.write(file, config_header())      ConfigDB @@ -278,11 +303,7 @@ defmodule Mix.Tasks.Pleroma.Config do      |> Enum.each(&write_and_delete(&1, file, opts[:delete]))      :ok = File.close(file) -    System.cmd("mix", ["format", config_path]) - -    shell_info( -      "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs" -    ) +    System.cmd("mix", ["format", path])    end    if Code.ensure_loaded?(Config.Reader) do diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 2403ed581..e7f4b67a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -8,10 +8,13 @@ defmodule Mix.Tasks.Pleroma.Database do    alias Pleroma.Object    alias Pleroma.Repo    alias Pleroma.User +    require Logger    require Pleroma.Constants +    import Ecto.Query    import Mix.Pleroma +    use Mix.Task    @shortdoc "A collection of database related tasks" @@ -214,4 +217,32 @@ defmodule Mix.Tasks.Pleroma.Database do        shell_info('Done.')      end    end + +  # Rolls back a specific migration (leaving subsequent migrations applied). +  # WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility. +  # Based on https://stackoverflow.com/a/53825840 +  def run(["rollback", version]) do +    prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?" + +    if shell_prompt(prompt, "n") in ~w(Yn Y y) do +      {_, result, _} = +        Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> +          version = String.to_integer(version) +          re = ~r/^#{version}_.*\.exs/ +          path = Ecto.Migrator.migrations_path(repo) + +          with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))}, +               {_, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))}, +               {_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do +            {:ok, "Reversed migration: #{file}"} +          else +            {:find, _} -> {:error, "No migration found with version prefix: #{version}"} +            {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"} +            {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"} +          end +        end) + +      shell_info(inspect(result)) +    end +  end  end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6542e684e..53beca5e6 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -113,6 +113,7 @@ defmodule Pleroma.Activity do      from([a] in query,        left_join: b in Bookmark,        on: b.user_id == ^user.id and b.activity_id == a.id, +      as: :bookmark,        preload: [bookmark: b]      )    end @@ -123,6 +124,7 @@ defmodule Pleroma.Activity do      from([a] in query,        left_join: r in ReportNote,        on: a.id == r.activity_id, +      as: :report_note,        preload: [report_notes: r]      )    end @@ -182,40 +184,48 @@ defmodule Pleroma.Activity do      |> Repo.one()    end -  @spec get_by_id(String.t()) :: Activity.t() | nil -  def get_by_id(id) do -    case FlakeId.flake_id?(id) do -      true -> -        Activity -        |> where([a], a.id == ^id) -        |> restrict_deactivated_users() -        |> Repo.one() - -      _ -> -        nil -    end -  end - -  def get_by_id_with_user_actor(id) do -    case FlakeId.flake_id?(id) do -      true -> -        Activity -        |> where([a], a.id == ^id) -        |> with_preloaded_user_actor() -        |> Repo.one() - -      _ -> -        nil +  @doc """ +  Gets activity by ID, doesn't load activities from deactivated actors by default. +  """ +  @spec get_by_id(String.t(), keyword()) :: t() | nil +  def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts) + +  @spec get_by_id_with_user_actor(String.t()) :: t() | nil +  def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor]) + +  @spec get_by_id_with_object(String.t()) :: t() | nil +  def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object]) + +  defp get_by_id_with_opts(id, opts) do +    if FlakeId.flake_id?(id) do +      query = Queries.by_id(id) + +      with_filters_query = +        if is_list(opts[:filter]) do +          Enum.reduce(opts[:filter], query, fn +            {:type, type}, acc -> Queries.by_type(acc, type) +            :restrict_deactivated, acc -> restrict_deactivated_users(acc) +            _, acc -> acc +          end) +        else +          query +        end + +      with_preloads_query = +        if is_list(opts[:preload]) do +          Enum.reduce(opts[:preload], with_filters_query, fn +            :user_actor, acc -> with_preloaded_user_actor(acc) +            :object, acc -> with_preloaded_object(acc) +            _, acc -> acc +          end) +        else +          with_filters_query +        end + +      Repo.one(with_preloads_query)      end    end -  def get_by_id_with_object(id) do -    Activity -    |> where(id: ^id) -    |> with_preloaded_object() -    |> Repo.one() -  end -    def all_by_ids_with_object(ids) do      Activity      |> where([a], a.id in ^ids) @@ -267,6 +277,11 @@ defmodule Pleroma.Activity do    def get_create_by_object_ap_id_with_object(_), do: nil +  @spec create_by_id_with_object(String.t()) :: t() | nil +  def create_by_id_with_object(id) do +    get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"]) +  end +    defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do      get_create_by_object_ap_id_with_object(ap_id)    end @@ -366,12 +381,6 @@ defmodule Pleroma.Activity do      end    end -  @spec pinned_by_actor?(Activity.t()) :: boolean() -  def pinned_by_actor?(%Activity{} = activity) do -    actor = user_actor(activity) -    activity.id in actor.pinned_activities -  end -    @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil    def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do      ap_id @@ -382,4 +391,13 @@ defmodule Pleroma.Activity do    end    def get_by_object_ap_id_with_object(_), do: nil + +  @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t() +  def add_by_params_query(object_id, actor, target) do +    object_id +    |> Queries.by_object_id() +    |> Queries.by_type("Add") +    |> Queries.by_actor(actor) +    |> where([a], fragment("?->>'target' = ?", a.data, ^target)) +  end  end diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index d94395fc1..7a603a615 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do      tags    end -  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do -    tags -    |> Enum.filter(&is_bitstring(&1)) -    |> Enum.map(fn tag -> "hashtag:" <> tag end) +  defp hashtags_to_topics(object) do +    object +    |> Object.hashtags() +    |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)    end -  defp hashtags_to_topics(_), do: [] -    defp remote_topics(%{local: true}), do: []    defp remote_topics(%{actor: actor}) when is_binary(actor), diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index a6b02a889..4632651b0 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do    alias Pleroma.Activity    alias Pleroma.User +  @spec by_id(query(), String.t()) :: query() +  def by_id(query \\ Activity, id) do +    from(a in query, where: a.id == ^id) +  end +    @spec by_ap_id(query, String.t()) :: query    def by_ap_id(query \\ Activity, ap_id) do      from( diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c853a2bb4..06d399b2e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -103,9 +103,7 @@ defmodule Pleroma.Application do          task_children(@mix_env) ++          dont_run_in_test(@mix_env) ++          chat_child(chat_enabled?()) ++ -        [ -          Pleroma.Gopher.Server -        ] +        [Pleroma.Gopher.Server]      # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html      # for other strategies and supported options @@ -230,6 +228,12 @@ defmodule Pleroma.Application do           keys: :duplicate,           partitions: System.schedulers_online()         ]} +    ] ++ background_migrators() +  end + +  defp background_migrators do +    [ +      Pleroma.Migrators.HashtagsTableMigrator      ]    end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 6ef65b263..ee6ee9516 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,15 +34,16 @@ defmodule Pleroma.ApplicationRequirements do    defp check_welcome_message_config!(:ok) do      if Pleroma.Config.get([:welcome, :email, :enabled], false) and           not Pleroma.Emails.Mailer.enabled?() do -      Logger.error(""" -      To send welcome email do you need to enable mail. -      \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true -      """) +      Logger.warn(""" +      To send welcome emails, you need to enable the mailer. +      Welcome emails will NOT be sent with the current config. -      {:error, "The mail disabled."} -    else -      :ok +      Enable the mailer: +        config :pleroma, Pleroma.Emails.Mailer, enabled: true +      """)      end + +    :ok    end    defp check_welcome_message_config!(result), do: result @@ -51,18 +52,21 @@ defmodule Pleroma.ApplicationRequirements do    #    def check_confirmation_accounts!(:ok) do      if Pleroma.Config.get([:instance, :account_activation_required]) && -         not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do -      Logger.error( -        "Account activation enabled, but no Mailer settings enabled.\n" <> -          "Please set config :pleroma, :instance, account_activation_required: false\n" <> -          "Otherwise setup and enable Mailer." -      ) +         not Pleroma.Emails.Mailer.enabled?() do +      Logger.warn(""" +      Account activation is required, but the mailer is disabled. +      Users will NOT be able to confirm their accounts with this config. +      Either disable account activation or enable the mailer. -      {:error, -       "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} -    else -      :ok +      Disable account activation: +        config :pleroma, :instance, account_activation_required: false + +      Enable the mailer: +        config :pleroma, Pleroma.Emails.Mailer, enabled: true +      """)      end + +    :ok    end    def check_confirmation_accounts!(result), do: result @@ -160,9 +164,11 @@ defmodule Pleroma.ApplicationRequirements do    defp check_system_commands!(:ok) do      filter_commands_statuses = [ -      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), -      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), -      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") +      check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"), +      check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), +      check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), +      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), +      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert")      ]      preview_proxy_commands_status = diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 2e15a3719..54e332595 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -99,4 +99,8 @@ defmodule Pleroma.Config do    def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], [])    def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] + +  def feature_enabled?(feature_name) do +    get([:features, feature_name]) not in [nil, false, :disabled, :auto] +  end  end diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 8227195dc..e5e9d3dcd 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -1,6 +1,6 @@  defmodule Pleroma.Config.ReleaseRuntimeProvider do    @moduledoc """ -  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases. +  Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.    """    @behaviour Config.Provider @@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do    def init(opts), do: opts    @impl true -  def load(config, _opts) do +  def load(config, opts) do      with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) -    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" +    config_path = +      opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"      with_runtime_config =        if File.exists?(config_path) do @@ -24,7 +25,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do          warning = [            IO.ANSI.red(),            IO.ANSI.bright(), -          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", +          "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",            IO.ANSI.reset()          ] @@ -33,13 +34,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do        end      exported_config_path = -      config_path -      |> Path.dirname() -      |> Path.join("prod.exported_from_db.secret.exs") +      opts[:exported_config_path] || +        config_path +        |> Path.dirname() +        |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")      with_exported =        if File.exists?(exported_config_path) do -        exported_config = Config.Reader.read!(with_runtime_config) +        exported_config = Config.Reader.read!(exported_config_path)          Config.Reader.merge(with_runtime_config, exported_config)        else          with_runtime_config diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index b874e0e37..cb57673e3 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -387,6 +387,6 @@ defmodule Pleroma.ConfigDB do    @spec module_name?(String.t()) :: boolean()    def module_name?(string) do      Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or -      string in ["Oban", "Ueberauth", "ExSyslogger"] +      string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]    end  end diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex new file mode 100644 index 000000000..1377af16e --- /dev/null +++ b/lib/pleroma/data_migration.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.DataMigration do +  use Ecto.Schema + +  alias Pleroma.DataMigration +  alias Pleroma.DataMigration.State +  alias Pleroma.Repo + +  import Ecto.Changeset +  import Ecto.Query + +  schema "data_migrations" do +    field(:name, :string) +    field(:state, State, default: :pending) +    field(:feature_lock, :boolean, default: false) +    field(:params, :map, default: %{}) +    field(:data, :map, default: %{}) + +    timestamps() +  end + +  def changeset(data_migration, params \\ %{}) do +    data_migration +    |> cast(params, [:name, :state, :feature_lock, :params, :data]) +    |> validate_required([:name]) +    |> unique_constraint(:name) +  end + +  def update_one_by_id(id, params \\ %{}) do +    with {1, _} <- +           from(dm in DataMigration, where: dm.id == ^id) +           |> Repo.update_all(set: params) do +      :ok +    end +  end + +  def get_by_name(name) do +    Repo.get_by(DataMigration, name: name) +  end + +  def populate_hashtags_table, do: get_by_name("populate_hashtags_table") +end diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index e8d536767..511d5cf58 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Delivery do    alias Pleroma.Object    alias Pleroma.Repo    alias Pleroma.User -  alias Pleroma.User    import Ecto.Changeset    import Ecto.Query diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex deleted file mode 100644 index 31cae3c72..000000000 --- a/lib/pleroma/earmark_renderer.ex +++ /dev/null @@ -1,256 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only -# -# This file is derived from Earmark, under the following copyright: -# Copyright © 2014 Dave Thomas, The Pragmatic Programmers -# SPDX-License-Identifier: Apache-2.0 -# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex -defmodule Pleroma.EarmarkRenderer do -  @moduledoc false - -  alias Earmark.Block -  alias Earmark.Context -  alias Earmark.HtmlRenderer -  alias Earmark.Options - -  import Earmark.Inline, only: [convert: 3] -  import Earmark.Helpers.HtmlHelpers -  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2] -  import Earmark.Context, only: [append: 2, set_value: 2] -  import Earmark.Options, only: [get_mapper: 1] - -  @doc false -  def render(blocks, %Context{options: %Options{}} = context) do -    messages = get_messages(context) - -    {contexts, html} = -      get_mapper(context.options).( -        blocks, -        &render_block(&1, put_in(context.options.messages, [])) -      ) -      |> Enum.unzip() - -    all_messages = -      contexts -      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end) - -    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()} -  end - -  ############# -  # Paragraph # -  ############# -  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do -    lines = convert(lines, lnb, context) -    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb) -  end - -  ######## -  # Html # -  ######## -  defp render_block(%Block.Html{html: html}, context) do -    {context, html} -  end - -  defp render_block(%Block.HtmlComment{lines: lines}, context) do -    {context, lines} -  end - -  defp render_block(%Block.HtmlOneline{html: html}, context) do -    {context, html} -  end - -  ######### -  # Ruler # -  ######### -  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do -    add_attrs(context, "<hr />", attrs, [], lnb) -  end - -  ########### -  # Heading # -  ########### -  defp render_block( -         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs}, -         context -       ) do -    converted = convert(content, lnb, context) -    html = "<h#{level}>#{converted.value}</h#{level}>" -    add_attrs(converted, html, attrs, [], lnb) -  end - -  ############## -  # Blockquote # -  ############## - -  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do -    {context1, body} = render(blocks, context) -    html = "<blockquote>#{body}</blockquote>" -    add_attrs(context1, html, attrs, [], lnb) -  end - -  ######### -  # Table # -  ######### - -  defp render_block( -         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs}, -         context -       ) do -    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb) -    context2 = set_value(context1, html) - -    context3 = -      if header do -        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>") -      else -        # Maybe an error, needed append(context, html) -        context2 -      end - -    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>") - -    {context4, [context4.value, "</table>"]} -  end - -  ######## -  # Code # -  ######## - -  defp render_block( -         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block, -         %Context{options: options} = context -       ) do -    class = -      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: "" - -    tag = ~s[<pre><code#{class}>] -    lines = options.render_code.(block) -    html = ~s[#{tag}#{lines}</code></pre>] -    add_attrs(context, html, attrs, [], lnb) -  end - -  ######### -  # Lists # -  ######### - -  defp render_block( -         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start}, -         context -       ) do -    {context1, content} = render(items, context) -    html = "<#{type}#{start}>#{content}</#{type}>" -    add_attrs(context1, html, attrs, [], lnb) -  end - -  # format a single paragraph list item, and remove the para tags -  defp render_block( -         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs}, -         context -       ) -       when length(blocks) == 1 do -    {context1, content} = render(blocks, context) -    content = Regex.replace(~r{</?p>}, content, "") -    html = "<li>#{content}</li>" -    add_attrs(context1, html, attrs, [], lnb) -  end - -  # format a spaced list item -  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do -    {context1, content} = render(blocks, context) -    html = "<li>#{content}</li>" -    add_attrs(context1, html, attrs, [], lnb) -  end - -  ################## -  # Footnote Block # -  ################## - -  defp render_block(%Block.FnList{blocks: footnotes}, context) do -    items = -      Enum.map(footnotes, fn note -> -        blocks = append_footnote_link(note) -        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks} -      end) - -    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context) -    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])} -  end - -  ####################################### -  # Isolated IALs are rendered as paras # -  ####################################### - -  defp render_block(%Block.Ial{verbatim: verbatim}, context) do -    {context, "<p>{:#{verbatim}}</p>"} -  end - -  #################### -  # IDDef is ignored # -  #################### - -  defp render_block(%Block.IdDef{}, context), do: {context, ""} - -  ##################################### -  # And here are the inline renderers # -  ##################################### - -  defdelegate br, to: HtmlRenderer -  defdelegate codespan(text), to: HtmlRenderer -  defdelegate em(text), to: HtmlRenderer -  defdelegate strong(text), to: HtmlRenderer -  defdelegate strikethrough(text), to: HtmlRenderer - -  defdelegate link(url, text), to: HtmlRenderer -  defdelegate link(url, text, title), to: HtmlRenderer - -  defdelegate image(path, alt, title), to: HtmlRenderer - -  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer - -  # Table rows -  defp add_trs(context, rows, tag, aligns, lnb) do -    numbered_rows = -      rows -      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1))) - -    numbered_rows -    |> Enum.reduce(context, fn {row, lnb}, ctx -> -      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>") -    end) -  end - -  defp add_tds(context, row, tag, aligns, lnb) do -    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb)) -  end - -  defp add_td_fn(row, tag, aligns, lnb) do -    fn n, ctx -> -      style = -        case Enum.at(aligns, n - 1, :default) do -          :default -> "" -          align -> " style=\"text-align: #{align}\"" -        end - -      col = Enum.at(row, n - 1) -      converted = convert(col, lnb, set_messages(ctx, [])) -      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>") -    end -  end - -  ############################### -  # Append Footnote Return Link # -  ############################### - -  defdelegate append_footnote_link(note), to: HtmlRenderer -  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer - -  defdelegate render_code(lines), to: HtmlRenderer - -  defp code_classes(language, prefix) do -    ["" | String.split(prefix || "")] -    |> Enum.map(fn pfx -> "#{pfx}#{language}" end) -    |> Enum.join(" ") -  end -end diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index f198cccb7..2a9addabc 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -17,3 +17,11 @@ defenum(Pleroma.FollowingRelationship.State,    follow_accept: 2,    follow_reject: 3  ) + +defenum(Pleroma.DataMigration.State, +  pending: 1, +  running: 2, +  complete: 3, +  failed: 4, +  manual: 5 +) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 7a08e48a9..764e347ec 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -121,6 +121,10 @@ defmodule Pleroma.Formatter do      end    end +  def markdown_to_html(text) do +    Earmark.as_html!(text, %Earmark.Options{compact_output: true}) +  end +    def html_escape({text, mentions, hashtags}, type) do      {html_escape(text, type), mentions, hashtags}    end diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex new file mode 100644 index 000000000..53e2e9c89 --- /dev/null +++ b/lib/pleroma/hashtag.ex @@ -0,0 +1,106 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Hashtag do +  use Ecto.Schema + +  import Ecto.Changeset +  import Ecto.Query + +  alias Ecto.Multi +  alias Pleroma.Hashtag +  alias Pleroma.Object +  alias Pleroma.Repo + +  schema "hashtags" do +    field(:name, :string) + +    many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete) + +    timestamps() +  end + +  def normalize_name(name) do +    name +    |> String.downcase() +    |> String.trim() +  end + +  def get_or_create_by_name(name) do +    changeset = changeset(%Hashtag{}, %{name: name}) + +    Repo.insert( +      changeset, +      on_conflict: [set: [name: get_field(changeset, :name)]], +      conflict_target: :name, +      returning: true +    ) +  end + +  def get_or_create_by_names(names) when is_list(names) do +    names = Enum.map(names, &normalize_name/1) +    timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + +    structs = +      Enum.map(names, fn name -> +        %Hashtag{} +        |> changeset(%{name: name}) +        |> Map.get(:changes) +        |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp}) +      end) + +    try do +      with {:ok, %{query_op: hashtags}} <- +             Multi.new() +             |> Multi.insert_all(:insert_all_op, Hashtag, structs, +               on_conflict: :nothing, +               conflict_target: :name +             ) +             |> Multi.run(:query_op, fn _repo, _changes -> +               {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} +             end) +             |> Repo.transaction() do +        {:ok, hashtags} +      else +        {:error, _name, value, _changes_so_far} -> {:error, value} +      end +    rescue +      e -> {:error, e} +    end +  end + +  def changeset(%Hashtag{} = struct, params) do +    struct +    |> cast(params, [:name]) +    |> update_change(:name, &normalize_name/1) +    |> validate_required([:name]) +    |> unique_constraint(:name) +  end + +  def unlink(%Object{id: object_id}) do +    with {_, hashtag_ids} <- +           from(hto in "hashtags_objects", +             where: hto.object_id == ^object_id, +             select: hto.hashtag_id +           ) +           |> Repo.delete_all(), +         {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do +      {:ok, length(hashtag_ids), unreferenced_count} +    end +  end + +  @delete_unreferenced_query """ +  DELETE FROM hashtags WHERE id IN +    (SELECT hashtags.id FROM hashtags +      LEFT OUTER JOIN hashtags_objects +        ON hashtags_objects.hashtag_id = hashtags.id +      WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1)); +  """ + +  def delete_unreferenced(ids) do +    with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do +      {:ok, deleted_count} +    end +  end +end diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex new file mode 100644 index 000000000..b84058e11 --- /dev/null +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -0,0 +1,208 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.HashtagsTableMigrator do +  defmodule State do +    use Pleroma.Migrators.Support.BaseMigratorState + +    @impl Pleroma.Migrators.Support.BaseMigratorState +    defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table +  end + +  use Pleroma.Migrators.Support.BaseMigrator + +  alias Pleroma.Hashtag +  alias Pleroma.Migrators.Support.BaseMigrator +  alias Pleroma.Object + +  @impl BaseMigrator +  def feature_config_path, do: [:features, :improved_hashtag_timeline] + +  @impl BaseMigrator +  def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0) + +  @impl BaseMigrator +  def perform do +    data_migration_id = data_migration_id() +    max_processed_id = get_stat(:max_processed_id, 0) + +    Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...") + +    query() +    |> where([object], object.id > ^max_processed_id) +    |> Repo.chunk_stream(100, :batches, timeout: :infinity) +    |> Stream.each(fn objects -> +      object_ids = Enum.map(objects, & &1.id) + +      results = Enum.map(objects, &transfer_object_hashtags(&1)) + +      failed_ids = +        results +        |> Enum.filter(&(elem(&1, 0) == :error)) +        |> Enum.map(&elem(&1, 1)) + +      # Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags +      chunk_affected_count = +        results +        |> Enum.filter(&(elem(&1, 0) == :ok)) +        |> length() + +      for failed_id <- failed_ids do +        _ = +          Repo.query( +            "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <> +              "VALUES ($1, $2) ON CONFLICT DO NOTHING;", +            [data_migration_id, failed_id] +          ) +      end + +      _ = +        Repo.query( +          "DELETE FROM data_migration_failed_ids " <> +            "WHERE data_migration_id = $1 AND record_id = ANY($2)", +          [data_migration_id, object_ids -- failed_ids] +        ) + +      max_object_id = Enum.at(object_ids, -1) + +      put_stat(:max_processed_id, max_object_id) +      increment_stat(:iteration_processed_count, length(object_ids)) +      increment_stat(:processed_count, length(object_ids)) +      increment_stat(:failed_count, length(failed_ids)) +      increment_stat(:affected_count, chunk_affected_count) +      put_stat(:records_per_second, records_per_second()) +      persist_state() + +      # A quick and dirty approach to controlling the load this background migration imposes +      sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0) +      Process.sleep(sleep_interval) +    end) +    |> Stream.run() +  end + +  @impl BaseMigrator +  def query do +    # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out) +    # Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up +    from( +      object in Object, +      where: +        fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data), +      select: %{ +        id: object.id, +        tag: fragment("(?)->'tag'", object.data) +      } +    ) +    |> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"), +      on: hashtags_objects.object_id == o.id +    ) +    |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id)) +  end + +  @spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()} +  defp transfer_object_hashtags(object) do +    embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"] +    hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) + +    if Enum.any?(hashtags) do +      transfer_object_hashtags(object, hashtags) +    else +      {:noop, object.id} +    end +  end + +  defp transfer_object_hashtags(object, hashtags) do +    Repo.transaction(fn -> +      with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do +        maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id}) +        base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}" + +        try do +          with {rows_count, _} when is_integer(rows_count) <- +                 Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do +            object.id +          else +            e -> +              Logger.error("#{base_error}: #{inspect(e)}") +              Repo.rollback(object.id) +          end +        rescue +          e -> +            Logger.error("#{base_error}: #{inspect(e)}") +            Repo.rollback(object.id) +        end +      else +        e -> +          error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" +          Logger.error(error) +          Repo.rollback(object.id) +      end +    end) +  end + +  @impl BaseMigrator +  def retry_failed do +    data_migration_id = data_migration_id() + +    failed_objects_query() +    |> Repo.chunk_stream(100, :one) +    |> Stream.each(fn object -> +      with {res, _} when res != :error <- transfer_object_hashtags(object) do +        _ = +          Repo.query( +            "DELETE FROM data_migration_failed_ids " <> +              "WHERE data_migration_id = $1 AND record_id = $2", +            [data_migration_id, object.id] +          ) +      end +    end) +    |> Stream.run() + +    put_stat(:failed_count, failures_count()) +    persist_state() + +    force_continue() +  end + +  defp failed_objects_query do +    from(o in Object) +    |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"), +      on: dmf.record_id == o.id +    ) +    |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id()) +    |> order_by([o], asc: o.id) +  end + +  @doc """ +  Service func to delete `hashtags_objects` for legacy objects not associated with Create activity. +  Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`). +  """ +  def delete_non_create_activities_hashtags do +    hashtags_objects_cleanup_query = """ +    DELETE FROM hashtags_objects WHERE object_id IN +      (SELECT DISTINCT objects.id FROM objects +        JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities +          ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = +            (objects.data->>'id') +          AND activities.data->>'type' = 'Create' +        WHERE activities.id IS NULL); +    """ + +    hashtags_cleanup_query = """ +    DELETE FROM hashtags WHERE id IN +      (SELECT hashtags.id FROM hashtags +        LEFT OUTER JOIN hashtags_objects +          ON hashtags_objects.hashtag_id = hashtags.id +        WHERE hashtags_objects.hashtag_id IS NULL); +    """ + +    {:ok, %{num_rows: hashtags_objects_count}} = +      Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity) + +    {:ok, %{num_rows: hashtags_count}} = +      Repo.query(hashtags_cleanup_query, [], timeout: :infinity) + +    {:ok, hashtags_objects_count, hashtags_count} +  end +end diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex new file mode 100644 index 000000000..1f8a5402b --- /dev/null +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -0,0 +1,210 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.Support.BaseMigrator do +  @moduledoc """ +  Base background migrator functionality. +  """ + +  @callback perform() :: any() +  @callback retry_failed() :: any() +  @callback feature_config_path() :: list(atom()) +  @callback query() :: Ecto.Query.t() +  @callback fault_rate_allowance() :: integer() | float() + +  defmacro __using__(_opts) do +    quote do +      use GenServer + +      require Logger + +      import Ecto.Query + +      alias __MODULE__.State +      alias Pleroma.Config +      alias Pleroma.Repo + +      @behaviour Pleroma.Migrators.Support.BaseMigrator + +      defdelegate data_migration(), to: State +      defdelegate data_migration_id(), to: State +      defdelegate state(), to: State +      defdelegate persist_state(), to: State, as: :persist_to_db +      defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key +      defdelegate put_stat(key, value), to: State, as: :put_data_key +      defdelegate increment_stat(key, increment), to: State, as: :increment_data_key + +      @reg_name {:global, __MODULE__} + +      def whereis, do: GenServer.whereis(@reg_name) + +      def start_link(_) do +        case whereis() do +          nil -> +            GenServer.start_link(__MODULE__, nil, name: @reg_name) + +          pid -> +            {:ok, pid} +        end +      end + +      @impl true +      def init(_) do +        {:ok, nil, {:continue, :init_state}} +      end + +      @impl true +      def handle_continue(:init_state, _state) do +        {:ok, _} = State.start_link(nil) + +        data_migration = data_migration() +        manual_migrations = Config.get([:instance, :manual_data_migrations], []) + +        cond do +          Config.get(:env) == :test -> +            update_status(:noop) + +          is_nil(data_migration) -> +            message = "Data migration does not exist." +            update_status(:failed, message) +            Logger.error("#{__MODULE__}: #{message}") + +          data_migration.state == :manual or data_migration.name in manual_migrations -> +            message = "Data migration is in manual execution or manual fix mode." +            update_status(:manual, message) +            Logger.warn("#{__MODULE__}: #{message}") + +          data_migration.state == :complete -> +            on_complete(data_migration) + +          true -> +            send(self(), :perform) +        end + +        {:noreply, nil} +      end + +      @impl true +      def handle_info(:perform, state) do +        State.reinit() + +        update_status(:running) +        put_stat(:iteration_processed_count, 0) +        put_stat(:started_at, NaiveDateTime.utc_now()) + +        perform() + +        fault_rate = fault_rate() +        put_stat(:fault_rate, fault_rate) +        fault_rate_allowance = fault_rate_allowance() + +        cond do +          fault_rate == 0 -> +            set_complete() + +          is_float(fault_rate) and fault_rate <= fault_rate_allowance -> +            message = """ +            Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}. +            Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`. +            """ + +            Logger.warn("#{__MODULE__}: #{message}") +            update_status(:manual, message) +            on_complete(data_migration()) + +          true -> +            message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`." +            Logger.error("#{__MODULE__}: #{message}") +            update_status(:failed, message) +        end + +        persist_state() +        {:noreply, state} +      end + +      defp on_complete(data_migration) do +        if data_migration.feature_lock || feature_state() == :disabled do +          Logger.warn( +            "#{__MODULE__}: migration complete but feature is locked; consider enabling." +          ) + +          :noop +        else +          Config.put(feature_config_path(), :enabled) +          :ok +        end +      end + +      @doc "Approximate count for current iteration (including processed records count)" +      def count(force \\ false, timeout \\ :infinity) do +        stored_count = get_stat(:count) + +        if stored_count && !force do +          stored_count +        else +          processed_count = get_stat(:processed_count, 0) +          max_processed_id = get_stat(:max_processed_id, 0) +          query = where(query(), [entity], entity.id > ^max_processed_id) + +          count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count +          put_stat(:count, count) +          persist_state() + +          count +        end +      end + +      def failures_count do +        with {:ok, %{rows: [[count]]}} <- +               Repo.query( +                 "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;", +                 [data_migration_id()] +               ) do +          count +        end +      end + +      def feature_state, do: Config.get(feature_config_path()) + +      def force_continue do +        send(whereis(), :perform) +      end + +      def force_restart do +        :ok = State.reset() +        force_continue() +      end + +      def set_complete do +        update_status(:complete) +        persist_state() +        on_complete(data_migration()) +      end + +      defp update_status(status, message \\ nil) do +        put_stat(:state, status) +        put_stat(:message, message) +      end + +      defp fault_rate do +        with failures_count when is_integer(failures_count) <- failures_count() do +          failures_count / Enum.max([get_stat(:affected_count, 0), 1]) +        else +          _ -> :error +        end +      end + +      defp records_per_second do +        get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1]) +      end + +      defp running_time do +        NaiveDateTime.diff( +          NaiveDateTime.utc_now(), +          get_stat(:started_at, NaiveDateTime.utc_now()) +        ) +      end +    end +  end +end diff --git a/lib/pleroma/migrators/support/base_migrator_state.ex b/lib/pleroma/migrators/support/base_migrator_state.ex new file mode 100644 index 000000000..b698587f2 --- /dev/null +++ b/lib/pleroma/migrators/support/base_migrator_state.ex @@ -0,0 +1,117 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Migrators.Support.BaseMigratorState do +  @moduledoc """ +  Base background migrator state functionality. +  """ + +  @callback data_migration() :: Pleroma.DataMigration.t() + +  defmacro __using__(_opts) do +    quote do +      use Agent + +      alias Pleroma.DataMigration + +      @behaviour Pleroma.Migrators.Support.BaseMigratorState +      @reg_name {:global, __MODULE__} + +      def start_link(_) do +        Agent.start_link(fn -> load_state_from_db() end, name: @reg_name) +      end + +      def data_migration, do: raise("data_migration/0 is not implemented") +      defoverridable data_migration: 0 + +      defp load_state_from_db do +        data_migration = data_migration() + +        data = +          if data_migration do +            Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end) +          else +            %{} +          end + +        %{ +          data_migration_id: data_migration && data_migration.id, +          data: data +        } +      end + +      def persist_to_db do +        %{data_migration_id: data_migration_id, data: data} = state() + +        if data_migration_id do +          DataMigration.update_one_by_id(data_migration_id, data: data) +        else +          {:error, :nil_data_migration_id} +        end +      end + +      def reset do +        %{data_migration_id: data_migration_id} = state() + +        with false <- is_nil(data_migration_id), +             :ok <- +               DataMigration.update_one_by_id(data_migration_id, +                 state: :pending, +                 data: %{} +               ) do +          reinit() +        else +          true -> {:error, :nil_data_migration_id} +          e -> e +        end +      end + +      def reinit do +        Agent.update(@reg_name, fn _state -> load_state_from_db() end) +      end + +      def state do +        Agent.get(@reg_name, & &1) +      end + +      def get_data_key(key, default \\ nil) do +        get_in(state(), [:data, key]) || default +      end + +      def put_data_key(key, value) do +        _ = persist_non_data_change(key, value) + +        Agent.update(@reg_name, fn state -> +          put_in(state, [:data, key], value) +        end) +      end + +      def increment_data_key(key, increment \\ 1) do +        Agent.update(@reg_name, fn state -> +          initial_value = get_in(state, [:data, key]) || 0 +          updated_value = initial_value + increment +          put_in(state, [:data, key], updated_value) +        end) +      end + +      defp persist_non_data_change(:state, value) do +        with true <- get_data_key(:state) != value, +             true <- value in Pleroma.DataMigration.State.__valid_values__(), +             %{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <- +               state() do +          DataMigration.update_one_by_id(data_migration_id, state: value) +        else +          false -> :ok +          _ -> {:error, :nil_data_migration_id} +        end +      end + +      defp persist_non_data_change(_, _) do +        nil +      end + +      def data_migration_id, do: Map.get(state(), :data_migration_id) +    end +  end +end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index aaf123840..3ba749d1a 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Object do    alias Pleroma.Activity    alias Pleroma.Config +  alias Pleroma.Hashtag    alias Pleroma.Object    alias Pleroma.Object.Fetcher    alias Pleroma.ObjectTombstone @@ -28,6 +29,8 @@ defmodule Pleroma.Object do    schema "objects" do      field(:data, :map) +    many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete) +      timestamps()    end @@ -49,7 +52,8 @@ defmodule Pleroma.Object do    end    def create(data) do -    Object.change(%Object{}, %{data: data}) +    %Object{} +    |> Object.change(%{data: data})      |> Repo.insert()    end @@ -58,8 +62,41 @@ defmodule Pleroma.Object do      |> cast(params, [:data])      |> validate_required([:data])      |> unique_constraint(:ap_id, name: :objects_unique_apid_index) +    # Expecting `maybe_handle_hashtags_change/1` to run last: +    |> maybe_handle_hashtags_change(struct) +  end + +  # Note: not checking activity type (assuming non-legacy objects are associated with Create act.) +  defp maybe_handle_hashtags_change(changeset, struct) do +    with %Ecto.Changeset{valid?: true} <- changeset, +         data_hashtags_change = get_change(changeset, :data), +         {_, true} <- {:changed, hashtags_changed?(struct, data_hashtags_change)}, +         {:ok, hashtag_records} <- +           data_hashtags_change +           |> object_data_hashtags() +           |> Hashtag.get_or_create_by_names() do +      put_assoc(changeset, :hashtags, hashtag_records) +    else +      %{valid?: false} -> +        changeset + +      {:changed, false} -> +        changeset + +      {:error, _} -> +        validate_change(changeset, :data, fn _, _ -> +          [data: "error referencing hashtags"] +        end) +    end +  end + +  defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do +    Enum.sort(embedded_hashtags(struct)) != +      Enum.sort(object_data_hashtags(data))    end +  defp hashtags_changed?(_, _), do: false +    def get_by_id(nil), do: nil    def get_by_id(id), do: Repo.get(Object, id) @@ -187,9 +224,13 @@ defmodule Pleroma.Object do    def swap_object_with_tombstone(object) do      tombstone = make_tombstone(object) -    object -    |> Object.change(%{data: tombstone}) -    |> Repo.update() +    with {:ok, object} <- +           object +           |> Object.change(%{data: tombstone}) +           |> Repo.update() do +      Hashtag.unlink(object) +      {:ok, object} +    end    end    def delete(%Object{data: %{"id" => id}} = object) do @@ -349,4 +390,39 @@ defmodule Pleroma.Object do    def self_replies(object, opts \\ []),      do: replies(object, Keyword.put(opts, :self_only, true)) + +  def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags + +  def tags(_), do: [] + +  def hashtags(%Object{} = object) do +    # Note: always using embedded hashtags regardless whether they are migrated to hashtags table +    #   (embedded hashtags stay in sync anyways, and we avoid extra joins and preload hassle) +    embedded_hashtags(object) +  end + +  def embedded_hashtags(%Object{data: data}) do +    object_data_hashtags(data) +  end + +  def embedded_hashtags(_), do: [] + +  def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do +    tags +    |> Enum.filter(fn +      %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name") +      plain_text when is_bitstring(plain_text) -> true +      _ -> false +    end) +    |> Enum.map(fn +      %{"name" => "#" <> hashtag} -> String.downcase(hashtag) +      %{"name" => hashtag} -> String.downcase(hashtag) +      hashtag when is_bitstring(hashtag) -> String.downcase(hashtag) +    end) +    |> Enum.uniq() +    # Note: "" elements (plain text) might occur in `data.tag` for incoming objects +    |> Enum.filter(&(&1 not in [nil, ""])) +  end + +  def object_data_hashtags(_), do: []  end diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index fb0398f92..040537acf 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -71,6 +71,14 @@ defmodule Pleroma.Object.Containment do      compare_uris(id_uri, other_uri)    end +  # Mastodon pin activities don't have an id, so we check the object field, which will be pinned. +  def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do +    id_uri = URI.parse(id) +    object_uri = URI.parse(object) + +    compare_uris(id_uri, object_uri) +  end +    def contain_origin_from_id(_id, _data), do: :error    def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}), diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 0d24e1010..33e45a0eb 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -93,6 +93,7 @@ defmodule Pleroma.Pagination do        max_id: :string,        offset: :integer,        limit: :integer, +      skip_extra_order: :boolean,        skip_order: :boolean      } @@ -114,6 +115,8 @@ defmodule Pleroma.Pagination do    defp restrict(query, :order, %{skip_order: true}, _), do: query +  defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query +    defp restrict(query, :order, %{min_id: _}, table_binding) do      order_by(        query, diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 4556352d0..b8ea06e33 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Repo do      adapter: Ecto.Adapters.Postgres,      migration_timestamps: [type: :naive_datetime_usec] +  use Ecto.Explain +    import Ecto.Query    require Logger @@ -63,8 +65,8 @@ defmodule Pleroma.Repo do    iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)    """    @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t() -  def chunk_stream(query, chunk_size, returns_as \\ :one) do -    # We don't actually need start and end funcitons of resource streaming, +  def chunk_stream(query, chunk_size, returns_as \\ :one, query_options \\ []) do +    # We don't actually need start and end functions of resource streaming,      # but it seems to be the only way to not fetch records one-by-one and      # have individual records be the elements of the stream, instead of      # lists of records @@ -76,7 +78,7 @@ defmodule Pleroma.Repo do            |> order_by(asc: :id)            |> where([r], r.id > ^last_id)            |> limit(^chunk_size) -          |> all() +          |> all(query_options)            |> case do              [] ->                {:halt, last_id} diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 654711351..5570ed104 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -23,6 +23,9 @@ defmodule Pleroma.Upload do      is once created permanent and changing it (especially in uploaders) is probably a bad idea!    * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the    path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. +  * `:width` - width of the media in pixels +  * `:height` - height of the media in pixels +  * `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/)    Related behaviors: @@ -32,6 +35,7 @@ defmodule Pleroma.Upload do    """    alias Ecto.UUID    alias Pleroma.Config +  alias Pleroma.Maps    require Logger    @type source :: @@ -53,9 +57,12 @@ defmodule Pleroma.Upload do            name: String.t(),            tempfile: String.t(),            content_type: String.t(), +          width: integer(), +          height: integer(), +          blurhash: String.t(),            path: String.t()          } -  defstruct [:id, :name, :tempfile, :content_type, :path] +  defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]    defp get_description(opts, upload) do      case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do @@ -89,9 +96,12 @@ defmodule Pleroma.Upload do               "mediaType" => upload.content_type,               "href" => url_from_spec(upload, opts.base_url, url_spec)             } +           |> Maps.put_if_present("width", upload.width) +           |> Maps.put_if_present("height", upload.height)           ],           "name" => description -       }} +       } +       |> Maps.put_if_present("blurhash", upload.blurhash)}      else        {:description_limit, _} ->          {:error, :description_too_long} diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex new file mode 100644 index 000000000..8c23076d4 --- /dev/null +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.AnalyzeMetadata do +  @moduledoc """ +  Extracts metadata about the upload, such as width/height +  """ +  require Logger + +  @behaviour Pleroma.Upload.Filter + +  @spec filter(Pleroma.Upload.t()) :: +          {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} +  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do +    try do +      image = +        file +        |> Mogrify.open() +        |> Mogrify.verbose() + +      upload = +        upload +        |> Map.put(:width, image.width) +        |> Map.put(:height, image.height) +        |> Map.put(:blurhash, get_blurhash(file)) + +      {:ok, :filtered, upload} +    rescue +      e in ErlangError -> +        Logger.warn("#{__MODULE__}: #{inspect(e)}") +        {:ok, :noop} +    end +  end + +  def filter(_), do: {:ok, :noop} + +  defp get_blurhash(file) do +    with {:ok, blurhash} <- :eblurhash.magick(file) do +      blurhash +    else +      _ -> nil +    end +  end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9942617d8..b78777141 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -99,6 +99,7 @@ defmodule Pleroma.User do      field(:local, :boolean, default: true)      field(:follower_address, :string)      field(:following_address, :string) +    field(:featured_address, :string)      field(:search_rank, :float, virtual: true)      field(:search_type, :integer, virtual: true)      field(:tags, {:array, :string}, default: []) @@ -130,7 +131,6 @@ defmodule Pleroma.User do      field(:hide_followers, :boolean, default: false)      field(:hide_follows, :boolean, default: false)      field(:hide_favorites, :boolean, default: true) -    field(:pinned_activities, {:array, :string}, default: [])      field(:email_notifications, :map, default: %{"digest" => false})      field(:mascot, :map, default: nil)      field(:emoji, :map, default: %{}) @@ -148,6 +148,7 @@ defmodule Pleroma.User do      field(:accepts_chat_messages, :boolean, default: nil)      field(:last_active_at, :naive_datetime)      field(:disclose_client, :boolean, default: true) +    field(:pinned_objects, :map, default: %{})      embeds_one(        :notification_settings, @@ -372,8 +373,10 @@ defmodule Pleroma.User do    end    # Should probably be renamed or removed +  @spec ap_id(User.t()) :: String.t()    def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" +  @spec ap_followers(User.t()) :: String.t()    def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa    def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" @@ -381,6 +384,11 @@ defmodule Pleroma.User do    def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa    def ap_following(%User{} = user), do: "#{ap_id(user)}/following" +  @spec ap_featured_collection(User.t()) :: String.t() +  def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa + +  def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured" +    defp truncate_fields_param(params) do      if Map.has_key?(params, :fields) do        Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) @@ -443,6 +451,7 @@ defmodule Pleroma.User do          :uri,          :follower_address,          :following_address, +        :featured_address,          :hide_followers,          :hide_follows,          :hide_followers_count, @@ -454,7 +463,8 @@ defmodule Pleroma.User do          :invisible,          :actor_type,          :also_known_as, -        :accepts_chat_messages +        :accepts_chat_messages, +        :pinned_objects        ]      )      |> cast(params, [:name], empty_values: []) @@ -686,7 +696,7 @@ defmodule Pleroma.User do      |> validate_format(:nickname, local_nickname_regex())      |> put_ap_id()      |> unique_constraint(:ap_id) -    |> put_following_and_follower_address() +    |> put_following_and_follower_and_featured_address()    end    def register_changeset(struct, params \\ %{}, opts \\ []) do @@ -747,7 +757,7 @@ defmodule Pleroma.User do      |> put_password_hash      |> put_ap_id()      |> unique_constraint(:ap_id) -    |> put_following_and_follower_address() +    |> put_following_and_follower_and_featured_address()    end    def maybe_validate_required_email(changeset, true), do: changeset @@ -765,11 +775,16 @@ defmodule Pleroma.User do      put_change(changeset, :ap_id, ap_id)    end -  defp put_following_and_follower_address(changeset) do -    followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) +  defp put_following_and_follower_and_featured_address(changeset) do +    user = %User{nickname: get_field(changeset, :nickname)} +    followers = ap_followers(user) +    following = ap_following(user) +    featured = ap_featured_collection(user)      changeset      |> put_change(:follower_address, followers) +    |> put_change(:following_address, following) +    |> put_change(:featured_address, featured)    end    defp autofollow_users(user) do @@ -2255,13 +2270,6 @@ defmodule Pleroma.User do      |> update_and_set_cache()    end -  def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do -    %{ -      admin: is_admin, -      moderator: is_moderator -    } -  end -    def validate_fields(changeset, remote? \\ false) do      limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields      limit = Config.get([:instance, limit_name], 0) @@ -2350,45 +2358,35 @@ defmodule Pleroma.User do      cast(user, %{is_approved: approved?}, [:is_approved])    end -  def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do -    if id not in user.pinned_activities do -      max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) -      params = %{pinned_activities: user.pinned_activities ++ [id]} - -      # if pinned activity was scheduled for deletion, we remove job -      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do -        Oban.cancel_job(expiration.id) -      end +  @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()} +  def add_pinned_object_id(%User{} = user, object_id) do +    if !user.pinned_objects[object_id] do +      params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())}        user -      |> cast(params, [:pinned_activities]) -      |> validate_length(:pinned_activities, -        max: max_pinned_statuses, -        message: "You have already pinned the maximum number of statuses" -      ) +      |> cast(params, [:pinned_objects]) +      |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects -> +        max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) + +        if Enum.count(pinned_objects) <= max_pinned_statuses do +          [] +        else +          [pinned_objects: "You have already pinned the maximum number of statuses"] +        end +      end)      else        change(user)      end      |> update_and_set_cache()    end -  def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do -    params = %{pinned_activities: List.delete(user.pinned_activities, id)} - -    # if pinned activity was scheduled for deletion, we reschedule it for deletion -    if data["expires_at"] do -      # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation -      {:ok, expires_at} = -        data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast() - -      Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ -        activity_id: id, -        expires_at: expires_at -      }) -    end - +  @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()} +  def remove_pinned_object_id(%User{} = user, object_id) do      user -    |> cast(params, [:pinned_activities]) +    |> cast( +      %{pinned_objects: Map.delete(user.pinned_objects, object_id)}, +      [:pinned_objects] +    )      |> update_and_set_cache()    end diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index bc0c95332..a446d3ae6 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Utils do      eperm epipe erange erofs espipe esrch estale etxtbsy exdev    )a +  @repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000) +    def compile_dir(dir) when is_binary(dir) do      dir      |> File.ls!() @@ -63,4 +65,21 @@ defmodule Pleroma.Utils do    end    def posix_error_message(_), do: "" + +  @doc """ +  Returns [timeout: integer] suitable for passing as an option to Repo functions. + +  This function detects if the execution was triggered from IEx shell, Mix task, or +  ./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value. +  """ +  @spec query_timeout() :: [timeout: integer] +  def query_timeout do +    {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2) + +    cond do +      parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] +      parent == :erl_eval -> [timeout: :infinity] +      true -> [timeout: @repo_timeout] +    end +  end  end diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index 8630f244b..397e4d1e7 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -35,9 +35,10 @@ defmodule Pleroma.Web do        import Plug.Conn        import Pleroma.Web.Gettext -      import Pleroma.Web.Router.Helpers        import Pleroma.Web.TranslationHelpers +      alias Pleroma.Web.Router.Helpers, as: Routes +        plug(:set_put_layout)        defp set_put_layout(conn, _) do @@ -131,7 +132,8 @@ defmodule Pleroma.Web do        import Pleroma.Web.ErrorHelpers        import Pleroma.Web.Gettext -      import Pleroma.Web.Router.Helpers + +      alias Pleroma.Web.Router.Helpers, as: Routes        require Logger @@ -233,16 +235,4 @@ defmodule Pleroma.Web do    def base_url do      Pleroma.Web.Endpoint.url()    end - -  # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ -  def get_api_routes do -    Pleroma.Web.Router.__routes__() -    |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) -    |> Enum.map(fn r -> -      r.path -      |> String.split("/", trim: true) -      |> List.first() -    end) -    |> Enum.uniq() -  end  end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5b45e2ca1..d0051d1cb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    alias Pleroma.Conversation    alias Pleroma.Conversation.Participation    alias Pleroma.Filter +  alias Pleroma.Hashtag    alias Pleroma.Maps    alias Pleroma.Notification    alias Pleroma.Object @@ -465,6 +466,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do      |> Repo.one()    end +  defp fetch_paginated_optimized(query, opts, pagination) do +    # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC", +    #   and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan +    opts = Map.put(opts, :skip_extra_order, true) + +    Pagination.fetch_paginated(query, opts, pagination) +  end + +  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do +    list_memberships = Pleroma.List.memberships(opts[:user]) + +    fetch_activities_query(recipients ++ list_memberships, opts) +    |> fetch_paginated_optimized(opts, pagination) +    |> Enum.reverse() +    |> maybe_update_cc(list_memberships, opts[:user]) +  end +    @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]    def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do      opts = Map.delete(opts, :user) @@ -472,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do      [Constants.as_public()]      |> fetch_activities_query(opts)      |> restrict_unlisted(opts) -    |> Pagination.fetch_paginated(opts, pagination) +    |> fetch_paginated_optimized(opts, pagination)    end    @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] @@ -612,7 +630,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do        |> Map.put(:type, ["Create", "Announce"])        |> Map.put(:user, reading_user)        |> Map.put(:actor_id, user.ap_id) -      |> Map.put(:pinned_activity_ids, user.pinned_activities) +      |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))      params =        if User.blocks?(reading_user, user) do @@ -693,51 +711,143 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    defp restrict_since(query, _), do: query -  defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do -    raise "Can't use the child object without preloading!" +  defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do +    raise_on_missing_preload() +  end + +  defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do +    from( +      [_activity, object] in query, +      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) +    ) +  end + +  defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do +    restrict_embedded_tag_any(query, %{tag: tag}) +  end + +  defp restrict_embedded_tag_all(query, _), do: query + +  defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do +    raise_on_missing_preload()    end -  defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do +  defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do +    from( +      [_activity, object] in query, +      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any) +    ) +  end + +  defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do +    restrict_embedded_tag_any(query, %{tag: [tag]}) +  end + +  defp restrict_embedded_tag_any(query, _), do: query + +  defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do +    raise_on_missing_preload() +  end + +  defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do      from(        [_activity, object] in query,        where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)      )    end -  defp restrict_tag_reject(query, _), do: query +  defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject}) +       when is_binary(tag_reject) do +    restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]}) +  end -  defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do -    raise "Can't use the child object without preloading!" +  defp restrict_embedded_tag_reject_any(query, _), do: query + +  defp object_ids_query_for_tags(tags) do +    from(hto in "hashtags_objects") +    |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id) +    |> where([hto, ht], ht.name in ^tags) +    |> select([hto], hto.object_id) +    |> distinct([hto], true) +  end + +  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do +    raise_on_missing_preload() +  end + +  defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do +    restrict_hashtag_any(query, %{tag: single_tag})    end -  defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do +  defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do      from(        [_activity, object] in query, -      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) +      where: +        fragment( +          """ +          (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects +            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?) +              AND hashtags_objects.object_id = ?) @> ? +          """, +          ^tags, +          object.id, +          ^tags +        )      )    end -  defp restrict_tag_all(query, _), do: query +  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do +    restrict_hashtag_all(query, %{tag_all: [tag]}) +  end -  defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do -    raise "Can't use the child object without preloading!" +  defp restrict_hashtag_all(query, _), do: query + +  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do +    raise_on_missing_preload()    end -  defp restrict_tag(query, %{tag: tag}) when is_list(tag) do +  defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do +    hashtag_ids = +      from(ht in Hashtag, where: ht.name in ^tags, select: ht.id) +      |> Repo.all() + +    # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan      from(        [_activity, object] in query, -      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) +      join: hto in "hashtags_objects", +      on: hto.object_id == object.id, +      where: hto.hashtag_id in ^hashtag_ids, +      distinct: [desc: object.id], +      order_by: [desc: object.id]      )    end -  defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do +  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do +    restrict_hashtag_any(query, %{tag: [tag]}) +  end + +  defp restrict_hashtag_any(query, _), do: query + +  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do +    raise_on_missing_preload() +  end + +  defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do      from(        [_activity, object] in query, -      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) +      where: object.id not in subquery(object_ids_query_for_tags(tags_reject))      )    end -  defp restrict_tag(query, _), do: query +  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do +    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]}) +  end + +  defp restrict_hashtag_reject_any(query, _), do: query + +  defp raise_on_missing_preload do +    raise "Can't use the child object without preloading!" +  end    defp restrict_recipients(query, [], _user), do: query @@ -965,8 +1075,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    defp restrict_unlisted(query, _), do: query -  defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do -    from(activity in query, where: activity.id in ^ids) +  defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do +    from( +      [activity, object: o] in query, +      where: +        fragment( +          "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)", +          activity.data, +          activity.data, +          activity.data, +          ^ids +        ) +    )    end    defp restrict_pinned(query, _), do: query @@ -1098,6 +1218,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    defp maybe_order(query, _), do: query +  defp normalize_fetch_activities_query_opts(opts) do +    Enum.reduce([:tag, :tag_all, :tag_reject], opts, fn key, opts -> +      case opts[key] do +        value when is_bitstring(value) -> +          Map.put(opts, key, Hashtag.normalize_name(value)) + +        value when is_list(value) -> +          normalized_value = +            value +            |> Enum.map(&Hashtag.normalize_name/1) +            |> Enum.uniq() + +          Map.put(opts, key, normalized_value) + +        _ -> +          opts +      end +    end) +  end +    defp fetch_activities_query_ap_ids_ops(opts) do      source_user = opts[:muting_user]      ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] @@ -1121,6 +1261,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    end    def fetch_activities_query(recipients, opts \\ %{}) do +    opts = normalize_fetch_activities_query_opts(opts) +      {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} =        fetch_activities_query_ap_ids_ops(opts) @@ -1128,50 +1270,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do        skip_thread_containment: Config.get([:instance, :skip_thread_containment])      } -    Activity -    |> maybe_preload_objects(opts) -    |> maybe_preload_bookmarks(opts) -    |> maybe_preload_report_notes(opts) -    |> maybe_set_thread_muted_field(opts) -    |> maybe_order(opts) -    |> restrict_recipients(recipients, opts[:user]) -    |> restrict_replies(opts) -    |> restrict_tag(opts) -    |> restrict_tag_reject(opts) -    |> restrict_tag_all(opts) -    |> restrict_since(opts) -    |> restrict_local(opts) -    |> restrict_remote(opts) -    |> restrict_actor(opts) -    |> restrict_type(opts) -    |> restrict_state(opts) -    |> restrict_favorited_by(opts) -    |> restrict_blocked(restrict_blocked_opts) -    |> restrict_muted(restrict_muted_opts) -    |> restrict_filtered(opts) -    |> restrict_media(opts) -    |> restrict_visibility(opts) -    |> restrict_thread_visibility(opts, config) -    |> restrict_reblogs(opts) -    |> restrict_pinned(opts) -    |> restrict_muted_reblogs(restrict_muted_reblogs_opts) -    |> restrict_instance(opts) -    |> restrict_announce_object_actor(opts) -    |> restrict_filtered(opts) -    |> Activity.restrict_deactivated_users() -    |> exclude_poll_votes(opts) -    |> exclude_chat_messages(opts) -    |> exclude_invisible_actors(opts) -    |> exclude_visibility(opts) -  end - -  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do -    list_memberships = Pleroma.List.memberships(opts[:user]) - -    fetch_activities_query(recipients ++ list_memberships, opts) -    |> Pagination.fetch_paginated(opts, pagination) -    |> Enum.reverse() -    |> maybe_update_cc(list_memberships, opts[:user]) +    query = +      Activity +      |> maybe_preload_objects(opts) +      |> maybe_preload_bookmarks(opts) +      |> maybe_preload_report_notes(opts) +      |> maybe_set_thread_muted_field(opts) +      |> maybe_order(opts) +      |> restrict_recipients(recipients, opts[:user]) +      |> restrict_replies(opts) +      |> restrict_since(opts) +      |> restrict_local(opts) +      |> restrict_remote(opts) +      |> restrict_actor(opts) +      |> restrict_type(opts) +      |> restrict_state(opts) +      |> restrict_favorited_by(opts) +      |> restrict_blocked(restrict_blocked_opts) +      |> restrict_muted(restrict_muted_opts) +      |> restrict_filtered(opts) +      |> restrict_media(opts) +      |> restrict_visibility(opts) +      |> restrict_thread_visibility(opts, config) +      |> restrict_reblogs(opts) +      |> restrict_pinned(opts) +      |> restrict_muted_reblogs(restrict_muted_reblogs_opts) +      |> restrict_instance(opts) +      |> restrict_announce_object_actor(opts) +      |> restrict_filtered(opts) +      |> Activity.restrict_deactivated_users() +      |> exclude_poll_votes(opts) +      |> exclude_chat_messages(opts) +      |> exclude_invisible_actors(opts) +      |> exclude_visibility(opts) + +    if Config.feature_enabled?(:improved_hashtag_timeline) do +      query +      |> restrict_hashtag_any(opts) +      |> restrict_hashtag_all(opts) +      |> restrict_hashtag_reject_any(opts) +    else +      query +      |> restrict_embedded_tag_any(opts) +      |> restrict_embedded_tag_all(opts) +      |> restrict_embedded_tag_reject_any(opts) +    end    end    @doc """ @@ -1250,21 +1393,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do    defp get_actor_url(_url), do: nil -  defp object_to_user_data(data) do -    avatar = -      data["icon"]["url"] && -        %{ -          "type" => "Image", -          "url" => [%{"href" => data["icon"]["url"]}] -        } +  defp normalize_image(%{"url" => url}) do +    %{ +      "type" => "Image", +      "url" => [%{"href" => url}] +    } +  end -    banner = -      data["image"]["url"] && -        %{ -          "type" => "Image", -          "url" => [%{"href" => data["image"]["url"]}] -        } +  defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image() +  defp normalize_image(_), do: nil +  defp object_to_user_data(data) do      fields =        data        |> Map.get("attachment", []) @@ -1290,6 +1429,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do      invisible = data["invisible"] || false      actor_type = data["type"] || "Person" +    featured_address = data["featured"] +    {:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address) +      public_key =        if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do          data["publicKey"]["publicKeyPem"] @@ -1308,23 +1450,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do        ap_id: data["id"],        uri: get_actor_url(data["url"]),        ap_enabled: true, -      banner: banner, +      banner: normalize_image(data["image"]),        fields: fields,        emoji: emojis,        is_locked: is_locked,        is_discoverable: is_discoverable,        invisible: invisible, -      avatar: avatar, +      avatar: normalize_image(data["icon"]),        name: data["name"],        follower_address: data["followers"],        following_address: data["following"], +      featured_address: featured_address,        bio: data["summary"] || "",        actor_type: actor_type,        also_known_as: Map.get(data, "alsoKnownAs", []),        public_key: public_key,        inbox: data["inbox"],        shared_inbox: shared_inbox, -      accepts_chat_messages: accepts_chat_messages +      accepts_chat_messages: accepts_chat_messages, +      pinned_objects: pinned_objects      }      # nickname can be nil because of virtual actors @@ -1462,6 +1606,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do      end    end +  def pin_data_from_featured_collection(%{ +        "type" => type, +        "orderedItems" => objects +      }) +      when type in ["OrderedCollection", "Collection"] do +    Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end) +  end + +  def fetch_and_prepare_featured_from_ap_id(nil) do +    {:ok, %{}} +  end + +  def fetch_and_prepare_featured_from_ap_id(ap_id) do +    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do +      {:ok, pin_data_from_featured_collection(data)} +    else +      e -> +        Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}") +        {:ok, %{}} +    end +  end + +  def pinned_fetch_task(nil), do: nil + +  def pinned_fetch_task(%{pinned_objects: pins}) do +    if Enum.all?(pins, fn {ap_id, _} -> +         Object.get_cached_by_ap_id(ap_id) || +           match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id)) +       end) do +      :ok +    else +      :error +    end +  end +    def make_user_from_ap_id(ap_id) do      user = User.get_cached_by_ap_id(ap_id) @@ -1469,6 +1648,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do        Transmogrifier.upgrade_user_from_ap_id(ap_id)      else        with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do +        {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) +          if user do            user            |> User.remote_user_changeset(data) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 9d3dcc7f9..5aa3b281a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -543,4 +543,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do        |> json(object.data)      end    end + +  def pinned(conn, %{"nickname" => nickname}) do +    with %User{} = user <- User.get_cached_by_nickname(nickname) do +      conn +      |> put_resp_header("content-type", "application/activity+json") +      |> json(UserView.render("featured.json", %{user: user})) +    end +  end  end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index f56bfc600..91a45836f 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -273,4 +273,36 @@ defmodule Pleroma.Web.ActivityPub.Builder do         "context" => object.data["context"]       }, []}    end + +  @spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()} +  def pin(%User{} = user, object) do +    {:ok, +     %{ +       "id" => Utils.generate_activity_id(), +       "target" => pinned_url(user.nickname), +       "object" => object.data["id"], +       "actor" => user.ap_id, +       "type" => "Add", +       "to" => [Pleroma.Constants.as_public()], +       "cc" => [user.follower_address] +     }, []} +  end + +  @spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()} +  def unpin(%User{} = user, object) do +    {:ok, +     %{ +       "id" => Utils.generate_activity_id(), +       "target" => pinned_url(user.nickname), +       "object" => object.data["id"], +       "actor" => user.ap_id, +       "type" => "Remove", +       "to" => [Pleroma.Constants.as_public()], +       "cc" => [user.follower_address] +     }, []} +  end + +  defp pinned_url(nickname) when is_binary(nickname) do +    Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname) +  end  end diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ef5a09a93..f2fec3ff6 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -92,7 +92,9 @@ defmodule Pleroma.Web.ActivityPub.MRF do    end    def get_policies do -    Pleroma.Config.get([:mrf, :policies], []) |> get_policies() +    Pleroma.Config.get([:mrf, :policies], []) +    |> get_policies() +    |> Enum.concat([Pleroma.Web.ActivityPub.MRF.HashtagPolicy])    end    defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex new file mode 100644 index 000000000..7307c9c14 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -0,0 +1,59 @@ +defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do +  @behaviour Pleroma.Web.ActivityPub.MRF +  alias Pleroma.Config +  alias Pleroma.User +  alias Pleroma.Web.CommonAPI + +  require Logger + +  @impl true +  def filter(message) do +    with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), +         %User{actor_type: "Service"} = follower <- +           User.get_cached_by_nickname(follower_nickname), +         %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do +      try_follow(follower, message) +    else +      nil -> +        Logger.warn( +          "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname +            account does not exist, or the account is not correctly configured as a bot." +        ) + +        {:ok, message} + +      _ -> +        {:ok, message} +    end +  end + +  defp try_follow(follower, message) do +    to = Map.get(message, "to", []) +    cc = Map.get(message, "cc", []) +    actor = [message["actor"]] + +    Enum.concat([to, cc, actor]) +    |> List.flatten() +    |> Enum.uniq() +    |> User.get_all_by_ap_id() +    |> Enum.each(fn user -> +      with false <- user.local, +           false <- User.following?(follower, user), +           false <- User.locked?(user), +           false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do +        Logger.debug( +          "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" +        ) + +        CommonAPI.follow(follower, user) +      end +    end) + +    {:ok, message} +  end + +  @impl true +  def describe do +    {:ok, %{}} +  end +end diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex new file mode 100644 index 000000000..def0c437c --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -0,0 +1,116 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do +  require Pleroma.Constants + +  alias Pleroma.Config +  alias Pleroma.Object + +  @moduledoc """ +  Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #) + +  Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. +  """ + +  @behaviour Pleroma.Web.ActivityPub.MRF + +  defp check_reject(message, hashtags) do +    if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do +      {:reject, "[HashtagPolicy] Matches with rejected keyword"} +    else +      {:ok, message} +    end +  end + +  defp check_ftl_removal(%{"to" => to} = message, hashtags) do +    if Pleroma.Constants.as_public() in to and +         Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match -> +           match in hashtags +         end) do +      to = List.delete(to, Pleroma.Constants.as_public()) +      cc = [Pleroma.Constants.as_public() | message["cc"] || []] + +      message = +        message +        |> Map.put("to", to) +        |> Map.put("cc", cc) +        |> Kernel.put_in(["object", "to"], to) +        |> Kernel.put_in(["object", "cc"], cc) + +      {:ok, message} +    else +      {:ok, message} +    end +  end + +  defp check_ftl_removal(message, _hashtags), do: {:ok, message} + +  defp check_sensitive(message, hashtags) do +    if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do +      {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} +    else +      {:ok, message} +    end +  end + +  @impl true +  def filter(%{"type" => "Create", "object" => object} = message) do +    hashtags = Object.hashtags(%Object{data: object}) + +    if hashtags != [] do +      with {:ok, message} <- check_reject(message, hashtags), +           {:ok, message} <- check_ftl_removal(message, hashtags), +           {:ok, message} <- check_sensitive(message, hashtags) do +        {:ok, message} +      end +    else +      {:ok, message} +    end +  end + +  @impl true +  def filter(message), do: {:ok, message} + +  @impl true +  def describe do +    mrf_hashtag = +      Config.get(:mrf_hashtag) +      |> Enum.into(%{}) + +    {:ok, %{mrf_hashtag: mrf_hashtag}} +  end + +  @impl true +  def config_description do +    %{ +      key: :mrf_hashtag, +      related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy", +      label: "MRF Hashtag", +      description: @moduledoc, +      children: [ +        %{ +          key: :reject, +          type: {:list, :string}, +          description: "A list of hashtags which result in message being rejected.", +          suggestions: ["foo"] +        }, +        %{ +          key: :federated_timeline_removal, +          type: {:list, :string}, +          description: +            "A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).", +          suggestions: ["foo"] +        }, +        %{ +          key: :sensitive, +          type: {:list, :string}, +          description: +            "A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)", +          suggestions: ["nsfw", "r18"] +        } +      ] +    } +  end +end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index bb3838d2c..d40348cb1 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -64,20 +64,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do           %{host: actor_host} = _actor_info,           %{             "type" => "Create", -           "object" => child_object +           "object" => %{} = _child_object           } = object -       ) -       when is_map(child_object) do +       ) do      media_nsfw =        Config.get([:mrf_simple, :media_nsfw])        |> MRF.subdomains_regex()      object =        if MRF.subdomain_match?(media_nsfw, actor_host) do -        tags = (child_object["tag"] || []) ++ ["nsfw"] -        child_object = Map.put(child_object, "tag", tags) -        child_object = Map.put(child_object, "sensitive", true) -        Map.put(object, "object", child_object) +        Kernel.put_in(object, ["object", "sensitive"], true)        else          object        end @@ -181,6 +177,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do    defp check_banner_removal(_actor_info, object), do: {:ok, object} +  defp check_object(%{"object" => object} = activity) do +    with {:ok, _object} <- filter(object) do +      {:ok, activity} +    end +  end + +  defp check_object(object), do: {:ok, object} +    @impl true    def filter(%{"type" => "Delete", "actor" => actor} = object) do      %{host: actor_host} = URI.parse(actor) @@ -206,7 +210,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do           {:ok, object} <- check_media_nsfw(actor_info, object),           {:ok, object} <- check_ftl_removal(actor_info, object),           {:ok, object} <- check_followers_only(actor_info, object), -         {:ok, object} <- check_report_removal(actor_info, object) do +         {:ok, object} <- check_report_removal(actor_info, object), +         {:ok, object} <- check_object(object) do        {:ok, object}      else        {:reject, nil} -> {:reject, "[SimplePolicy]"} @@ -231,6 +236,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do      end    end +  def filter(object) when is_binary(object) do +    uri = URI.parse(object) + +    with {:ok, object} <- check_accept(uri, object), +         {:ok, object} <- check_reject(uri, object) do +      {:ok, object} +    else +      {:reject, nil} -> {:reject, "[SimplePolicy]"} +      {:reject, _} = e -> e +      _ -> {:reject, "[SimplePolicy]"} +    end +  end +    def filter(object), do: {:ok, object}    @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 5739cee63..528093ac0 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -28,20 +28,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do           "mrf_tag:media-force-nsfw",           %{             "type" => "Create", -           "object" => %{"attachment" => child_attachment} = object +           "object" => %{"attachment" => child_attachment}           } = message         )         when length(child_attachment) > 0 do -    tags = (object["tag"] || []) ++ ["nsfw"] - -    object = -      object -      |> Map.put("tag", tags) -      |> Map.put("sensitive", true) - -    message = Map.put(message, "object", object) - -    {:ok, message} +    {:ok, Kernel.put_in(message, ["object", "sensitive"], true)}    end    defp process_tag( diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 297c19cc0..1dce33f1a 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do    alias Pleroma.Object.Containment    alias Pleroma.User    alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator +  alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator @@ -37,37 +38,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do    @impl true    def validate(object, meta) -  def validate(%{"type" => type} = object, meta) -      when type in ~w[Accept Reject] do -    with {:ok, object} <- -           object -           |> AcceptRejectValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "Event"} = object, meta) do -    with {:ok, object} <- -           object -           |> EventValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "Follow"} = object, meta) do -    with {:ok, object} <- -           object -           |> FollowValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end -    def validate(%{"type" => "Block"} = block_activity, meta) do      with {:ok, block_activity} <-             block_activity @@ -87,16 +57,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do      end    end -  def validate(%{"type" => "Update"} = update_activity, meta) do -    with {:ok, update_activity} <- -           update_activity -           |> UpdateValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      update_activity = stringify_keys(update_activity) -      {:ok, update_activity, meta} -    end -  end -    def validate(%{"type" => "Undo"} = object, meta) do      with {:ok, object} <-             object @@ -123,76 +83,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do      end    end -  def validate(%{"type" => "Like"} = object, meta) do -    with {:ok, object} <- -           object -           |> LikeValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "ChatMessage"} = object, meta) do -    with {:ok, object} <- -           object -           |> ChatMessageValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "Question"} = object, meta) do -    with {:ok, object} <- -           object -           |> QuestionValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do -    with {:ok, object} <- -           object -           |> AudioVideoValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "Article"} = object, meta) do -    with {:ok, object} <- -           object -           |> ArticleNoteValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "Answer"} = object, meta) do -    with {:ok, object} <- -           object -           |> AnswerValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end - -  def validate(%{"type" => "EmojiReact"} = object, meta) do -    with {:ok, object} <- -           object -           |> EmojiReactValidator.cast_and_validate() -           |> Ecto.Changeset.apply_action(:insert) do -      object = stringify_keys(object) -      {:ok, object, meta} -    end -  end -    def validate(          %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,          meta @@ -224,10 +114,60 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do      end    end -  def validate(%{"type" => "Announce"} = object, meta) do +  def validate(%{"type" => type} = object, meta) +      when type in ~w[Event Question Audio Video Article] do +    validator = +      case type do +        "Event" -> EventValidator +        "Question" -> QuestionValidator +        "Audio" -> AudioVideoValidator +        "Video" -> AudioVideoValidator +        "Article" -> ArticleNoteValidator +      end + +    with {:ok, object} <- +           object +           |> validator.cast_and_validate() +           |> Ecto.Changeset.apply_action(:insert) do +      object = stringify_keys(object) + +      # Insert copy of hashtags as strings for the non-hashtag table indexing +      tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) +      object = Map.put(object, "tag", tag) + +      {:ok, object, meta} +    end +  end + +  def validate(%{"type" => type} = object, meta) +      when type in ~w[Accept Reject Follow Update Like EmojiReact Announce +      ChatMessage Answer] do +    validator = +      case type do +        "Accept" -> AcceptRejectValidator +        "Reject" -> AcceptRejectValidator +        "Follow" -> FollowValidator +        "Update" -> UpdateValidator +        "Like" -> LikeValidator +        "EmojiReact" -> EmojiReactValidator +        "Announce" -> AnnounceValidator +        "ChatMessage" -> ChatMessageValidator +        "Answer" -> AnswerValidator +      end + +    with {:ok, object} <- +           object +           |> validator.cast_and_validate() +           |> Ecto.Changeset.apply_action(:insert) do +      object = stringify_keys(object) +      {:ok, object, meta} +    end +  end + +  def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do      with {:ok, object} <-             object -           |> AnnounceValidator.cast_and_validate() +           |> AddRemoveValidator.cast_and_validate()             |> Ecto.Changeset.apply_action(:insert) do        object = stringify_keys(object)        {:ok, object, meta} @@ -260,7 +200,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do    def cast_and_apply(o), do: {:error, {:validator_not_set, o}} -  # is_struct/1 isn't present in Elixir 1.8.x +  # is_struct/1 appears in Elixir 1.11    def stringify_keys(%{__struct__: _} = object) do      object      |> Map.from_struct() diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex index d31e780c3..b577a1044 100644 --- a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_required([:id, :type, :actor, :to, :cc, :object])      |> validate_inclusion(:type, ["Accept", "Reject"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex new file mode 100644 index 000000000..f885aabe4 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do +  use Ecto.Schema + +  import Ecto.Changeset +  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + +  require Pleroma.Constants + +  alias Pleroma.EctoType.ActivityPub.ObjectValidators +  alias Pleroma.User + +  @primary_key false + +  embedded_schema do +    field(:id, ObjectValidators.ObjectID, primary_key: true) +    field(:target) +    field(:object, ObjectValidators.ObjectID) +    field(:actor, ObjectValidators.ObjectID) +    field(:type) +    field(:to, ObjectValidators.Recipients, default: []) +    field(:cc, ObjectValidators.Recipients, default: []) +  end + +  def cast_and_validate(data) do +    {:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"]) + +    {:ok, actor} = maybe_refetch_user(actor) + +    data +    |> maybe_fix_data_for_mastodon(actor) +    |> cast_data() +    |> validate_data(actor) +  end + +  defp maybe_fix_data_for_mastodon(data, actor) do +    # Mastodon sends pin/unpin objects without id, to, cc fields +    data +    |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) +    |> Map.put_new("to", [Pleroma.Constants.as_public()]) +    |> Map.put_new("cc", [actor.follower_address]) +  end + +  defp cast_data(data) do +    cast(%__MODULE__{}, data, __schema__(:fields)) +  end + +  defp validate_data(changeset, actor) do +    changeset +    |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) +    |> validate_inclusion(:type, ~w(Add Remove)) +    |> validate_actor_presence() +    |> validate_collection_belongs_to_actor(actor) +    |> validate_object_presence() +  end + +  defp validate_collection_belongs_to_actor(changeset, actor) do +    validate_change(changeset, :target, fn :target, target -> +      if target == actor.featured_address do +        [] +      else +        [target: "collection doesn't belong to actor"] +      end +    end) +  end + +  defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do +    {:ok, user} +  end + +  defp maybe_refetch_user(%User{ap_id: ap_id}) do +    Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id) +  end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index b08a33e68..576341790 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do      cng    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Announce"])      |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 15e4413cd..c9bd9e42d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Answer"])      |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index b0388ef3b..39ef6dc29 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do    alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations +  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator    alias Pleroma.Web.ActivityPub.Transmogrifier    import Ecto.Changeset @@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do      field(:cc, ObjectValidators.Recipients, default: [])      field(:bto, ObjectValidators.Recipients, default: [])      field(:bcc, ObjectValidators.Recipients, default: []) -    # TODO: Write type -    field(:tag, {:array, :map}, default: []) +    embeds_many(:tag, TagValidator)      field(:type, :string)      field(:name, :string) @@ -90,11 +90,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do      data = fix(data)      struct -    |> cast(data, __schema__(:fields) -- [:attachment]) +    |> cast(data, __schema__(:fields) -- [:attachment, :tag])      |> cast_embed(:attachment) +    |> cast_embed(:tag)    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Article", "Note"])      |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 3175427ad..bba2f5eb0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do    use Ecto.Schema    alias Pleroma.EctoType.ActivityPub.ObjectValidators -  alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator    import Ecto.Changeset @@ -21,6 +20,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do        field(:type, :string)        field(:href, ObjectValidators.Uri)        field(:mediaType, :string, default: "application/octet-stream") +      field(:width, :integer) +      field(:height, :integer)      end    end @@ -52,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do      data = fix_media_type(data)      struct -    |> cast(data, [:type, :href, :mediaType]) +    |> cast(data, [:type, :href, :mediaType, :width, :height])      |> validate_inclusion(:type, ["Link"])      |> validate_required([:type, :href, :mediaType])    end @@ -90,7 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do      end    end -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_inclusion(:type, ~w[Document Audio Image Video])      |> validate_required([:mediaType, :url, :type]) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 4a96fef52..27e14b16d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -5,11 +5,11 @@  defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do    use Ecto.Schema -  alias Pleroma.EarmarkRenderer    alias Pleroma.EctoType.ActivityPub.ObjectValidators    alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations +  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator    alias Pleroma.Web.ActivityPub.Transmogrifier    import Ecto.Changeset @@ -23,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do      field(:cc, ObjectValidators.Recipients, default: [])      field(:bto, ObjectValidators.Recipients, default: [])      field(:bcc, ObjectValidators.Recipients, default: []) -    # TODO: Write type -    field(:tag, {:array, :map}, default: []) +    embeds_many(:tag, TagValidator)      field(:type, :string)      field(:name, :string) @@ -110,7 +109,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do         when is_binary(content) do      content =        content -      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) +      |> Pleroma.Formatter.markdown_to_html()        |> Pleroma.HTML.filter_tags()      Map.put(data, "content", content) @@ -132,11 +131,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do      data = fix(data)      struct -    |> cast(data, __schema__(:fields) -- [:attachment]) +    |> cast(data, __schema__(:fields) -- [:attachment, :tag])      |> cast_embed(:attachment) +    |> cast_embed(:tag)    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Audio", "Video"])      |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index c5f77bb76..88948135f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_required([:id, :type, :actor, :to, :cc, :object])      |> validate_inclusion(:type, ["Block"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 1189778f2..b153156b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do      |> cast_embed(:attachment)    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["ChatMessage"])      |> validate_required([:id, :actor, :to, :type, :published]) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 093549a45..940430588 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do    alias Pleroma.Object    alias Pleroma.User +  @spec validate_any_presence(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()    def validate_any_presence(cng, fields) do      non_empty =        fields @@ -29,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      end    end +  @spec validate_actor_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()    def validate_actor_presence(cng, options \\ []) do      field_name = Keyword.get(options, :field_name, :actor) @@ -47,6 +49,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      end)    end +  @spec validate_object_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()    def validate_object_presence(cng, options \\ []) do      field_name = Keyword.get(options, :field_name, :object)      allowed_types = Keyword.get(options, :allowed_types, false) @@ -68,6 +71,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      end)    end +  @spec validate_object_or_user_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()    def validate_object_or_user_presence(cng, options \\ []) do      field_name = Keyword.get(options, :field_name, :object)      options = Keyword.put(options, :field_name, field_name) @@ -83,6 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      if actor_cng.valid?, do: actor_cng, else: object_cng    end +  @spec validate_host_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()    def validate_host_match(cng, fields \\ [:id, :actor]) do      if same_domain?(cng, fields) do        cng @@ -95,6 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      end    end +  @spec validate_fields_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()    def validate_fields_match(cng, fields) do      if map_unique?(cng, fields) do        cng @@ -122,12 +128,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do      end)    end +  @spec same_domain?(Ecto.Changeset.t(), [atom()]) :: boolean()    def same_domain?(cng, fields \\ [:actor, :object]) do      map_unique?(cng, fields, fn value -> URI.parse(value).host end)    end    # This figures out if a user is able to create, delete or modify something    # based on the domain and superuser status +  @spec validate_modification_rights(Ecto.Changeset.t()) :: Ecto.Changeset.t()    def validate_modification_rights(cng) do      actor = User.get_cached_by_ap_id(get_field(cng, :actor)) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex index 8384c16a7..7a31a99bf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do      |> validate_data(meta)    end -  def validate_data(cng, meta \\ []) do +  defp validate_data(cng, meta) do      cng      |> validate_required([:id, :actor, :to, :type, :object])      |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index bf56a918c..e06e442f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -79,7 +79,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do      |> CommonFixes.fix_actor()    end -  def validate_data(cng, meta \\ []) do +  defp validate_data(cng, meta) do      cng      |> validate_required([:actor, :type, :object])      |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index fc1a79a72..7da67bf16 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do      Tombstone      Video    } -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_required([:id, :type, :actor, :to, :cc, :object])      |> validate_inclusion(:type, ["Delete"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index 1906e597e..ec7566515 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do      end    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["EmojiReact"])      |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content]) diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 2e26726f8..d42458ef5 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do    alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations +  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator    alias Pleroma.Web.ActivityPub.Transmogrifier    import Ecto.Changeset @@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do      field(:cc, ObjectValidators.Recipients, default: [])      field(:bto, ObjectValidators.Recipients, default: [])      field(:bcc, ObjectValidators.Recipients, default: []) -    # TODO: Write type -    field(:tag, {:array, :map}, default: []) +    embeds_many(:tag, TagValidator)      field(:type, :string)      field(:name, :string) @@ -81,11 +81,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do      data = fix(data)      struct -    |> cast(data, __schema__(:fields) -- [:attachment]) +    |> cast(data, __schema__(:fields) -- [:attachment, :tag])      |> cast_embed(:attachment) +    |> cast_embed(:tag)    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Event"])      |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index 6e428bacc..239cee5e7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_required([:id, :type, :actor, :to, :cc, :object])      |> validate_inclusion(:type, ["Follow"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 30c40b238..509da507b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do      end    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Like"])      |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 6b746c997..7012e2e1d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes    alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations    alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator +  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator    alias Pleroma.Web.ActivityPub.Transmogrifier    import Ecto.Changeset @@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do      field(:cc, ObjectValidators.Recipients, default: [])      field(:bto, ObjectValidators.Recipients, default: [])      field(:bcc, ObjectValidators.Recipients, default: []) -    # TODO: Write type -    field(:tag, {:array, :map}, default: []) +    embeds_many(:tag, TagValidator)      field(:type, :string)      field(:content, :string)      field(:context, :string) @@ -93,13 +93,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do      data = fix(data)      struct -    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment]) +    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag])      |> cast_embed(:attachment)      |> cast_embed(:anyOf)      |> cast_embed(:oneOf) +    |> cast_embed(:tag)    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Question"])      |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex new file mode 100644 index 000000000..751021585 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do +  use Ecto.Schema + +  alias Pleroma.EctoType.ActivityPub.ObjectValidators + +  import Ecto.Changeset + +  @primary_key false +  embedded_schema do +    # Common +    field(:type, :string) +    field(:name, :string) + +    # Mention, Hashtag +    field(:href, ObjectValidators.Uri) + +    # Emoji +    embeds_one :icon, IconObjectValidator, primary_key: false do +      field(:type, :string) +      field(:url, ObjectValidators.Uri) +    end + +    field(:updated, ObjectValidators.DateTime) +    field(:id, ObjectValidators.Uri) +  end + +  def cast_and_validate(data) do +    data +    |> cast_data() +  end + +  def cast_data(data) do +    %__MODULE__{} +    |> changeset(data) +  end + +  def changeset(struct, %{"type" => "Mention"} = data) do +    struct +    |> cast(data, [:type, :name, :href]) +    |> validate_required([:type, :href]) +  end + +  def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do +    name = +      cond do +        "#" <> name -> name +        name -> name +      end +      |> String.downcase() + +    data = Map.put(data, "name", name) + +    struct +    |> cast(data, [:type, :name, :href]) +    |> validate_required([:type, :name]) +  end + +  def changeset(struct, %{"type" => "Emoji"} = data) do +    data = Map.put(data, "name", String.trim(data["name"], ":")) + +    struct +    |> cast(data, [:type, :name, :updated, :id]) +    |> cast_embed(:icon, with: &icon_changeset/2) +    |> validate_required([:type, :name, :icon]) +  end + +  def icon_changeset(struct, data) do +    struct +    |> cast(data, [:type, :url]) +    |> validate_inclusion(:type, ~w[Image]) +    |> validate_required([:type, :url]) +  end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex index 783a79ddb..e8af60ffa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(data_cng) do +  defp validate_data(data_cng) do      data_cng      |> validate_inclusion(:type, ["Undo"])      |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a66d41400..6bb1dc7fa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do      |> cast(data, __schema__(:fields))    end -  def validate_data(cng) do +  defp validate_data(cng) do      cng      |> validate_required([:id, :type, :actor, :to, :cc, :object])      |> validate_inclusion(:type, ["Update"]) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 195596f94..a0f2e0312 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do    alias Pleroma.Config    alias Pleroma.Object    alias Pleroma.Repo +  alias Pleroma.Utils    alias Pleroma.Web.ActivityPub.ActivityPub    alias Pleroma.Web.ActivityPub.MRF    alias Pleroma.Web.ActivityPub.ObjectValidator @@ -24,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do    @spec common_pipeline(map(), keyword()) ::            {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}    def common_pipeline(object, meta) do -    case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do +    case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do        {:ok, {:ok, activity, meta}} ->          @side_effects.handle_after_transaction(meta)          {:ok, activity, meta} @@ -40,19 +41,17 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do      end    end -  def do_common_pipeline(object, meta) do -    with {_, {:ok, validated_object, meta}} <- -           {:validate_object, @object_validator.validate(object, meta)}, -         {_, {:ok, mrfd_object, meta}} <- -           {:mrf_object, @mrf.pipeline_filter(validated_object, meta)}, -         {_, {:ok, activity, meta}} <- -           {:persist_object, @activity_pub.persist(mrfd_object, meta)}, -         {_, {:ok, activity, meta}} <- -           {:execute_side_effects, @side_effects.handle(activity, meta)}, -         {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do -      {:ok, activity, meta} +  def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct} + +  def do_common_pipeline(message, meta) do +    with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)}, +         {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)}, +         {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)}, +         {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)}, +         {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do +      {:ok, message, meta}      else -      {:mrf_object, {:reject, message, _}} -> {:reject, message} +      {:mrf, {:reject, message, _}} -> {:reject, message}        e -> {:error, e}      end    end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0b9a9f0c5..5fe143c2b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -276,10 +276,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do      result =        case deleted_object do          %Object{} -> -          with {:ok, deleted_object, activity} <- Object.delete(deleted_object), +          with {:ok, deleted_object, _activity} <- Object.delete(deleted_object),                 {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]},                 %User{} = user <- User.get_cached_by_ap_id(actor) do -            User.remove_pinnned_activity(user, activity) +            User.remove_pinned_object_id(user, deleted_object.data["id"])              {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object) @@ -312,6 +312,63 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do      end    end +  # Tasks this handles: +  # - adds pin to user +  # - removes expiration job for pinned activity, if was set for expiration +  @impl true +  def handle(%{data: %{"type" => "Add"} = data} = object, meta) do +    with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), +         {:ok, _user} <- User.add_pinned_object_id(user, data["object"]) do +      # if pinned activity was scheduled for deletion, we remove job +      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(meta[:activity_id]) do +        Oban.cancel_job(expiration.id) +      end + +      {:ok, object, meta} +    else +      nil -> +        {:error, :user_not_found} + +      {:error, changeset} -> +        if changeset.errors[:pinned_objects] do +          {:error, :pinned_statuses_limit_reached} +        else +          changeset.errors +        end +    end +  end + +  # Tasks this handles: +  # - removes pin from user +  # - removes corresponding Add activity +  # - if activity had expiration, recreates activity expiration job +  @impl true +  def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do +    with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), +         {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do +      data["object"] +      |> Activity.add_by_params_query(user.ap_id, user.featured_address) +      |> Repo.delete_all() + +      # if pinned activity was scheduled for deletion, we reschedule it for deletion +      if meta[:expires_at] do +        # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation +        {:ok, expires_at} = +          Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at]) + +        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ +          activity_id: meta[:activity_id], +          expires_at: expires_at +        }) +      end + +      {:ok, object, meta} +    else +      nil -> {:error, :user_not_found} +      error -> error +    end +  end +    # Nothing to do    @impl true    def handle(object, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4d9a5617e..a40d51c45 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -32,18 +32,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do    """    def fix_object(object, options \\ []) do      object -    |> strip_internal_fields -    |> fix_actor -    |> fix_url -    |> fix_attachments -    |> fix_context +    |> strip_internal_fields() +    |> fix_actor() +    |> fix_url() +    |> fix_attachments() +    |> fix_context()      |> fix_in_reply_to(options) -    |> fix_emoji -    |> fix_tag -    |> set_sensitive -    |> fix_content_map -    |> fix_addressing -    |> fix_summary +    |> fix_emoji() +    |> fix_tag() +    |> fix_content_map() +    |> fix_addressing() +    |> fix_summary()      |> fix_type(options)    end @@ -245,6 +244,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do                "type" => Map.get(url || %{}, "type", "Link")              }              |> Maps.put_if_present("mediaType", media_type) +            |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) +            |> Maps.put_if_present("height", (url || %{})["height"] || data["height"])            %{              "url" => [attachment_url], @@ -315,10 +316,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do      tags =        tag        |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) -      |> Enum.map(fn %{"name" => name} -> -        name -        |> String.slice(1..-1) -        |> String.downcase() +      |> Enum.map(fn +        %{"name" => "#" <> hashtag} -> String.downcase(hashtag) +        %{"name" => hashtag} -> String.downcase(hashtag)        end)      Map.put(object, "tag", tag ++ tags) @@ -536,7 +536,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do    end    def handle_incoming(%{"type" => type} = data, _options) -      when type in ~w{Like EmojiReact Announce} do +      when type in ~w{Like EmojiReact Announce Add Remove} do      with :ok <- ObjectValidator.fetch_actor_and_object(data),           {:ok, activity, _meta} <-             Pipeline.common_pipeline(data, local: false) do @@ -566,7 +566,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do             Pipeline.common_pipeline(data, local: false) do        {:ok, activity}      else -      {:error, {:validate_object, _}} = e -> +      {:error, {:validate, _}} = e ->          # Check if we have a create activity for this          with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),               %Activity{data: %{"actor" => actor}} <- @@ -742,7 +742,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do    # Prepares the object of an outgoing create activity.    def prepare_object(object) do      object -    |> set_sensitive      |> add_hashtags      |> add_mention_tags      |> add_emoji_tags @@ -933,15 +932,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do      Map.put(object, "conversation", object["context"])    end -  def set_sensitive(%{"sensitive" => _} = object) do -    object -  end - -  def set_sensitive(object) do -    tags = object["tag"] || [] -    Map.put(object, "sensitive", "nsfw" in tags) -  end -    def set_type(%{"type" => "Answer"} = object) do      Map.put(object, "type", "Note")    end @@ -961,7 +951,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do        object        |> Map.get("attachment", [])        |> Enum.map(fn data -> -        [%{"mediaType" => media_type, "href" => href} | _] = data["url"] +        [%{"mediaType" => media_type, "href" => href} = url | _] = data["url"]          %{            "url" => href, @@ -969,6 +959,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do            "name" => data["name"],            "type" => "Document"          } +        |> Maps.put_if_present("width", url["width"]) +        |> Maps.put_if_present("height", url["height"]) +        |> Maps.put_if_present("blurhash", data["blurhash"])        end)      Map.put(object, "attachment", attachments) @@ -1012,6 +1005,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do      with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),           {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),           {:ok, user} <- update_user(user, data) do +      {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end)        TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})        {:ok, user}      else diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 8adc9878a..344da19d3 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do    use Pleroma.Web, :view    alias Pleroma.Keys +  alias Pleroma.Object    alias Pleroma.Repo    alias Pleroma.User +  alias Pleroma.Web.ActivityPub.ObjectView    alias Pleroma.Web.ActivityPub.Transmogrifier    alias Pleroma.Web.ActivityPub.Utils    alias Pleroma.Web.Endpoint @@ -97,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do        "followers" => "#{user.ap_id}/followers",        "inbox" => "#{user.ap_id}/inbox",        "outbox" => "#{user.ap_id}/outbox", +      "featured" => "#{user.ap_id}/collections/featured",        "preferredUsername" => user.nickname,        "name" => user.name,        "summary" => user.bio, @@ -245,6 +248,25 @@ defmodule Pleroma.Web.ActivityPub.UserView do      |> Map.merge(pagination)    end +  def render("featured.json", %{ +        user: %{featured_address: featured_address, pinned_objects: pinned_objects} +      }) do +    objects = +      pinned_objects +      |> Enum.sort_by(fn {_, pinned_at} -> pinned_at end, &>=/2) +      |> Enum.map(fn {id, _} -> +        ObjectView.render("object.json", %{object: Object.get_cached_by_ap_id(id)}) +      end) + +    %{ +      "id" => featured_address, +      "type" => "OrderedCollection", +      "orderedItems" => objects, +      "totalItems" => length(objects) +    } +    |> Map.merge(Utils.make_json_ld_header()) +  end +    defp maybe_put_total_items(map, false, _total), do: map    defp maybe_put_total_items(map, true, total) do diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 65bc63cb9..d3e4c18a3 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -13,16 +13,17 @@ defmodule Pleroma.Web.AdminAPI.UserController do    alias Pleroma.Web.ActivityPub.Builder    alias Pleroma.Web.ActivityPub.Pipeline    alias Pleroma.Web.AdminAPI -  alias Pleroma.Web.AdminAPI.AccountView    alias Pleroma.Web.AdminAPI.Search    alias Pleroma.Web.Plugs.OAuthScopesPlug    @users_page_size 50 +  plug(Pleroma.Web.ApiSpec.CastAndValidate) +    plug(      OAuthScopesPlug,      %{scopes: ["admin:read:accounts"]} -    when action in [:list, :show] +    when action in [:index, :show]    )    plug( @@ -44,13 +45,19 @@ defmodule Pleroma.Web.AdminAPI.UserController do      when action in [:follow, :unfollow]    ) +  plug(:put_view, Pleroma.Web.AdminAPI.AccountView) +    action_fallback(AdminAPI.FallbackController) -  def delete(conn, %{"nickname" => nickname}) do -    delete(conn, %{"nicknames" => [nickname]}) +  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.UserOperation + +  def delete(conn, %{nickname: nickname}) do +    conn +    |> Map.put(:body_params, %{nicknames: [nickname]}) +    |> delete(%{})    end -  def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do +  def delete(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do      users = Enum.map(nicknames, &User.get_cached_by_nickname/1)      Enum.each(users, fn user -> @@ -67,10 +74,16 @@ defmodule Pleroma.Web.AdminAPI.UserController do      json(conn, nicknames)    end -  def follow(%{assigns: %{user: admin}} = conn, %{ -        "follower" => follower_nick, -        "followed" => followed_nick -      }) do +  def follow( +        %{ +          assigns: %{user: admin}, +          body_params: %{ +            follower: follower_nick, +            followed: followed_nick +          } +        } = conn, +        _ +      ) do      with %User{} = follower <- User.get_cached_by_nickname(follower_nick),           %User{} = followed <- User.get_cached_by_nickname(followed_nick) do        User.follow(follower, followed) @@ -86,10 +99,16 @@ defmodule Pleroma.Web.AdminAPI.UserController do      json(conn, "ok")    end -  def unfollow(%{assigns: %{user: admin}} = conn, %{ -        "follower" => follower_nick, -        "followed" => followed_nick -      }) do +  def unfollow( +        %{ +          assigns: %{user: admin}, +          body_params: %{ +            follower: follower_nick, +            followed: followed_nick +          } +        } = conn, +        _ +      ) do      with %User{} = follower <- User.get_cached_by_nickname(follower_nick),           %User{} = followed <- User.get_cached_by_nickname(followed_nick) do        User.unfollow(follower, followed) @@ -105,9 +124,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do      json(conn, "ok")    end -  def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do +  def create(%{assigns: %{user: admin}, body_params: %{users: users}} = conn, _) do      changesets = -      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> +      users +      |> Enum.map(fn %{nickname: nickname, email: email, password: password} ->          user_data = %{            nickname: nickname,            name: nickname, @@ -124,52 +144,49 @@ defmodule Pleroma.Web.AdminAPI.UserController do        end)      case Pleroma.Repo.transaction(changesets) do -      {:ok, users} -> -        res = -          users +      {:ok, users_map} -> +        users = +          users_map            |> Map.values()            |> Enum.map(fn user ->              {:ok, user} = User.post_register_action(user)              user            end) -          |> Enum.map(&AccountView.render("created.json", %{user: &1}))          ModerationLog.insert_log(%{            actor: admin, -          subjects: Map.values(users), +          subjects: users,            action: "create"          }) -        json(conn, res) +        render(conn, "created_many.json", users: users)        {:error, id, changeset, _} -> -        res = +        changesets =            Enum.map(changesets.operations, fn -            {current_id, {:changeset, _current_changeset, _}} when current_id == id -> -              AccountView.render("create-error.json", %{changeset: changeset}) +            {^id, {:changeset, _current_changeset, _}} -> +              changeset              {_, {:changeset, current_changeset, _}} -> -              AccountView.render("create-error.json", %{changeset: current_changeset}) +              current_changeset            end)          conn          |> put_status(:conflict) -        |> json(res) +        |> render("create_errors.json", changesets: changesets)      end    end -  def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do +  def show(%{assigns: %{user: admin}} = conn, %{nickname: nickname}) do      with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do -      conn -      |> put_view(AccountView) -      |> render("show.json", %{user: user}) +      render(conn, "show.json", %{user: user})      else        _ -> {:error, :not_found}      end    end -  def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do +  def toggle_activation(%{assigns: %{user: admin}} = conn, %{nickname: nickname}) do      user = User.get_cached_by_nickname(nickname)      {:ok, updated_user} = User.set_activation(user, !user.is_active) @@ -182,12 +199,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do        action: action      }) -    conn -    |> put_view(AccountView) -    |> render("show.json", %{user: updated_user}) +    render(conn, "show.json", user: updated_user)    end -  def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do +  def activate(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do      users = Enum.map(nicknames, &User.get_cached_by_nickname/1)      {:ok, updated_users} = User.set_activation(users, true) @@ -197,12 +212,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do        action: "activate"      }) -    conn -    |> put_view(AccountView) -    |> render("index.json", %{users: Keyword.values(updated_users)}) +    render(conn, "index.json", users: Keyword.values(updated_users))    end -  def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do +  def deactivate(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do      users = Enum.map(nicknames, &User.get_cached_by_nickname/1)      {:ok, updated_users} = User.set_activation(users, false) @@ -212,12 +225,10 @@ defmodule Pleroma.Web.AdminAPI.UserController do        action: "deactivate"      }) -    conn -    |> put_view(AccountView) -    |> render("index.json", %{users: Keyword.values(updated_users)}) +    render(conn, "index.json", users: Keyword.values(updated_users))    end -  def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do +  def approve(%{assigns: %{user: admin}, body_params: %{nicknames: nicknames}} = conn, _) do      users = Enum.map(nicknames, &User.get_cached_by_nickname/1)      {:ok, updated_users} = User.approve(users) @@ -227,36 +238,27 @@ defmodule Pleroma.Web.AdminAPI.UserController do        action: "approve"      }) -    conn -    |> put_view(AccountView) -    |> render("index.json", %{users: updated_users}) +    render(conn, "index.json", users: updated_users)    end -  def list(conn, params) do +  def index(conn, params) do      {page, page_size} = page_params(params) -    filters = maybe_parse_filters(params["filters"]) +    filters = maybe_parse_filters(params[:filters])      search_params =        %{ -        query: params["query"], +        query: params[:query],          page: page,          page_size: page_size, -        tags: params["tags"], -        name: params["name"], -        email: params["email"], -        actor_types: params["actor_types"] +        tags: params[:tags], +        name: params[:name], +        email: params[:email], +        actor_types: params[:actor_types]        }        |> Map.merge(filters)      with {:ok, users, count} <- Search.user(search_params) do -      json( -        conn, -        AccountView.render("index.json", -          users: users, -          count: count, -          page_size: page_size -        ) -      ) +      render(conn, "index.json", users: users, count: count, page_size: page_size)      end    end @@ -274,8 +276,8 @@ defmodule Pleroma.Web.AdminAPI.UserController do    defp page_params(params) do      { -      fetch_integer_param(params, "page", 1), -      fetch_integer_param(params, "page_size", @users_page_size) +      fetch_integer_param(params, :page, 1), +      fetch_integer_param(params, :page_size, @users_page_size)      }    end  end diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index d7c63d385..e053a9b67 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do        "display_name" => display_name,        "is_active" => user.is_active,        "local" => user.local, -      "roles" => User.roles(user), +      "roles" => roles(user),        "tags" => user.tags || [],        "is_confirmed" => user.is_confirmed,        "is_approved" => user.is_approved, @@ -85,6 +85,10 @@ defmodule Pleroma.Web.AdminAPI.AccountView do      }    end +  def render("created_many.json", %{users: users}) do +    render_many(users, AccountView, "created.json", as: :user) +  end +    def render("created.json", %{user: user}) do      %{        type: "success", @@ -96,7 +100,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do      }    end -  def render("create-error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do +  def render("create_errors.json", %{changesets: changesets}) do +    render_many(changesets, AccountView, "create_error.json", as: :changeset) +  end + +  def render("create_error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do      %{        type: "error",        code: 409, @@ -140,4 +148,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do    defp image_url(%{"url" => [%{"href" => href} | _]}), do: href    defp image_url(_), do: nil + +  defp roles(%{is_moderator: is_moderator, is_admin: is_admin}) do +    %{ +      admin: is_admin, +      moderator: is_moderator +    } +  end  end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index adc8762dc..528cd9cf4 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -92,9 +92,10 @@ defmodule Pleroma.Web.ApiSpec do                "Invites",                "MediaProxy cache",                "OAuth application managment", -              "Report managment",                "Relays", -              "Status administration" +              "Report managment", +              "Status administration", +              "User administration"              ]            },            %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index a3da856ff..d23a7dcb6 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do    @behaviour Plug +  alias OpenApiSpex.Plug.PutApiSpec    alias Plug.Conn    @impl Plug @@ -25,12 +26,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do    end    @impl Plug -  def call(%{private: %{open_api_spex: private_data}} = conn, %{ -        operation_id: operation_id, -        render_error: render_error -      }) do -    spec = private_data.spec -    operation = private_data.operation_lookup[operation_id] + +  def call(conn, %{operation_id: operation_id, render_error: render_error}) do +    {spec, operation_lookup} = PutApiSpec.get_spec_and_operation_lookup(conn) +    operation = operation_lookup[operation_id]      content_type =        case Conn.get_req_header(conn, "content-type") do @@ -43,8 +42,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do            "application/json"        end -    private_data = Map.put(private_data, :operation_id, operation_id) -    conn = Conn.put_private(conn, :open_api_spex, private_data) +    conn = Conn.put_private(conn, :operation_id, operation_id)      case cast_and_validate(spec, operation, conn, content_type, strict?()) do        {:ok, conn} -> @@ -64,25 +62,22 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do            private: %{              phoenix_controller: controller,              phoenix_action: action, -            open_api_spex: private_data +            open_api_spex: %{spec_module: spec_module}            }          } = conn,          opts        ) do +    {spec, operation_lookup} = PutApiSpec.get_spec_and_operation_lookup(conn) +      operation = -      case private_data.operation_lookup[{controller, action}] do +      case operation_lookup[{controller, action}] do          nil ->            operation_id = controller.open_api_operation(action).operationId -          operation = private_data.operation_lookup[operation_id] +          operation = operation_lookup[operation_id] -          operation_lookup = -            private_data.operation_lookup -            |> Map.put({controller, action}, operation) +          operation_lookup = Map.put(operation_lookup, {controller, action}, operation) -          OpenApiSpex.Plug.Cache.adapter().put( -            private_data.spec_module, -            {private_data.spec, operation_lookup} -          ) +          OpenApiSpex.Plug.Cache.adapter().put(spec_module, {spec, operation_lookup})            operation diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex new file mode 100644 index 000000000..c9d0bfd7c --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -0,0 +1,389 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do +  alias OpenApiSpex.Operation +  alias OpenApiSpex.Schema +  alias Pleroma.Web.ApiSpec.Schemas.ActorType +  alias Pleroma.Web.ApiSpec.Schemas.ApiError + +  import Pleroma.Web.ApiSpec.Helpers + +  def open_api_operation(action) do +    operation = String.to_existing_atom("#{action}_operation") +    apply(__MODULE__, operation, []) +  end + +  def index_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "List users", +      operationId: "AdminAPI.UserController.index", +      security: [%{"oAuth" => ["admin:read:accounts"]}], +      parameters: [ +        Operation.parameter(:filters, :query, :string, "Comma separated list of filters"), +        Operation.parameter(:query, :query, :string, "Search users query"), +        Operation.parameter(:name, :query, :string, "Search by display name"), +        Operation.parameter(:email, :query, :string, "Search by email"), +        Operation.parameter(:page, :query, :integer, "Page Number"), +        Operation.parameter(:page_size, :query, :integer, "Number of users to return per page"), +        Operation.parameter( +          :actor_types, +          :query, +          %Schema{type: :array, items: ActorType}, +          "Filter by actor type" +        ), +        Operation.parameter( +          :tags, +          :query, +          %Schema{type: :array, items: %Schema{type: :string}}, +          "Filter by tags" +        ) +        | admin_api_params() +      ], +      responses: %{ +        200 => +          Operation.response( +            "Response", +            "application/json", +            %Schema{ +              type: :object, +              properties: %{ +                users: %Schema{type: :array, items: user()}, +                count: %Schema{type: :integer}, +                page_size: %Schema{type: :integer} +              } +            } +          ), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def create_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Create a single or multiple users", +      operationId: "AdminAPI.UserController.create", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            description: "POST body for creating users", +            type: :object, +            properties: %{ +              users: %Schema{ +                type: :array, +                items: %Schema{ +                  type: :object, +                  properties: %{ +                    nickname: %Schema{type: :string}, +                    email: %Schema{type: :string}, +                    password: %Schema{type: :string} +                  } +                } +              } +            } +          } +        ), +      responses: %{ +        200 => +          Operation.response("Response", "application/json", %Schema{ +            type: :array, +            items: %Schema{ +              type: :object, +              properties: %{ +                code: %Schema{type: :integer}, +                type: %Schema{type: :string}, +                data: %Schema{ +                  type: :object, +                  properties: %{ +                    email: %Schema{type: :string, format: :email}, +                    nickname: %Schema{type: :string} +                  } +                } +              } +            } +          }), +        403 => Operation.response("Forbidden", "application/json", ApiError), +        409 => +          Operation.response("Conflict", "application/json", %Schema{ +            type: :array, +            items: %Schema{ +              type: :object, +              properties: %{ +                code: %Schema{type: :integer}, +                error: %Schema{type: :string}, +                type: %Schema{type: :string}, +                data: %Schema{ +                  type: :object, +                  properties: %{ +                    email: %Schema{type: :string, format: :email}, +                    nickname: %Schema{type: :string} +                  } +                } +              } +            } +          }) +      } +    } +  end + +  def show_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Show user", +      operationId: "AdminAPI.UserController.show", +      security: [%{"oAuth" => ["admin:read:accounts"]}], +      parameters: [ +        Operation.parameter( +          :nickname, +          :path, +          :string, +          "User nickname or ID" +        ) +        | admin_api_params() +      ], +      responses: %{ +        200 => Operation.response("Response", "application/json", user()), +        403 => Operation.response("Forbidden", "application/json", ApiError), +        404 => Operation.response("Not Found", "application/json", ApiError) +      } +    } +  end + +  def follow_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Follow", +      operationId: "AdminAPI.UserController.follow", +      security: [%{"oAuth" => ["admin:write:follows"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            type: :object, +            properties: %{ +              follower: %Schema{type: :string, description: "Follower nickname"}, +              followed: %Schema{type: :string, description: "Followed nickname"} +            } +          } +        ), +      responses: %{ +        200 => Operation.response("Response", "application/json", %Schema{type: :string}), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def unfollow_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Unfollow", +      operationId: "AdminAPI.UserController.unfollow", +      security: [%{"oAuth" => ["admin:write:follows"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            type: :object, +            properties: %{ +              follower: %Schema{type: :string, description: "Follower nickname"}, +              followed: %Schema{type: :string, description: "Followed nickname"} +            } +          } +        ), +      responses: %{ +        200 => Operation.response("Response", "application/json", %Schema{type: :string}), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def approve_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Approve multiple users", +      operationId: "AdminAPI.UserController.approve", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            description: "POST body for deleting multiple users", +            type: :object, +            properties: %{ +              nicknames: %Schema{ +                type: :array, +                items: %Schema{type: :string} +              } +            } +          } +        ), +      responses: %{ +        200 => +          Operation.response("Response", "application/json", %Schema{ +            type: :object, +            properties: %{user: %Schema{type: :array, items: user()}} +          }), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def toggle_activation_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Toggle user activation", +      operationId: "AdminAPI.UserController.toggle_activation", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: [ +        Operation.parameter(:nickname, :path, :string, "User nickname") +        | admin_api_params() +      ], +      responses: %{ +        200 => Operation.response("Response", "application/json", user()), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def activate_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Activate multiple users", +      operationId: "AdminAPI.UserController.activate", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            description: "POST body for deleting multiple users", +            type: :object, +            properties: %{ +              nicknames: %Schema{ +                type: :array, +                items: %Schema{type: :string} +              } +            } +          } +        ), +      responses: %{ +        200 => +          Operation.response("Response", "application/json", %Schema{ +            type: :object, +            properties: %{user: %Schema{type: :array, items: user()}} +          }), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def deactivate_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Deactivates multiple users", +      operationId: "AdminAPI.UserController.deactivate", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: admin_api_params(), +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            description: "POST body for deleting multiple users", +            type: :object, +            properties: %{ +              nicknames: %Schema{ +                type: :array, +                items: %Schema{type: :string} +              } +            } +          } +        ), +      responses: %{ +        200 => +          Operation.response("Response", "application/json", %Schema{ +            type: :object, +            properties: %{user: %Schema{type: :array, items: user()}} +          }), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  def delete_operation do +    %Operation{ +      tags: ["User administration"], +      summary: "Removes a single or multiple users", +      operationId: "AdminAPI.UserController.delete", +      security: [%{"oAuth" => ["admin:write:accounts"]}], +      parameters: [ +        Operation.parameter( +          :nickname, +          :query, +          :string, +          "User nickname" +        ) +        | admin_api_params() +      ], +      requestBody: +        request_body( +          "Parameters", +          %Schema{ +            description: "POST body for deleting multiple users", +            type: :object, +            properties: %{ +              nicknames: %Schema{ +                type: :array, +                items: %Schema{type: :string} +              } +            } +          } +        ), +      responses: %{ +        200 => +          Operation.response("Response", "application/json", %Schema{ +            description: "Array of nicknames", +            type: :array, +            items: %Schema{type: :string} +          }), +        403 => Operation.response("Forbidden", "application/json", ApiError) +      } +    } +  end + +  defp user do +    %Schema{ +      type: :object, +      properties: %{ +        id: %Schema{type: :string}, +        email: %Schema{type: :string, format: :email}, +        avatar: %Schema{type: :string, format: :uri}, +        nickname: %Schema{type: :string}, +        display_name: %Schema{type: :string}, +        is_active: %Schema{type: :boolean}, +        local: %Schema{type: :boolean}, +        roles: %Schema{ +          type: :object, +          properties: %{ +            admin: %Schema{type: :boolean}, +            moderator: %Schema{type: :boolean} +          } +        }, +        tags: %Schema{type: :array, items: %Schema{type: :string}}, +        is_confirmed: %Schema{type: :boolean}, +        is_approved: %Schema{type: :boolean}, +        url: %Schema{type: :string, format: :uri}, +        registration_reason: %Schema{type: :string, nullable: true}, +        actor_type: %Schema{type: :string} +      } +    } +  end +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 40edc747d..802fbef3e 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do            Operation.response(              "Status. When `scheduled_at` is present, ScheduledStatus is returned instead",              "application/json", -            %Schema{oneOf: [Status, ScheduledStatus]} +            %Schema{anyOf: [Status, ScheduledStatus]}            ),          422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)        } @@ -182,7 +182,34 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do        parameters: [id_param()],        responses: %{          200 => status_response(), -        400 => Operation.response("Error", "application/json", ApiError) +        400 => +          Operation.response("Bad Request", "application/json", %Schema{ +            allOf: [ApiError], +            title: "Unprocessable Entity", +            example: %{ +              "error" => "You have already pinned the maximum number of statuses" +            } +          }), +        404 => +          Operation.response("Not found", "application/json", %Schema{ +            allOf: [ApiError], +            title: "Unprocessable Entity", +            example: %{ +              "error" => "Record not found" +            } +          }), +        422 => +          Operation.response( +            "Unprocessable Entity", +            "application/json", +            %Schema{ +              allOf: [ApiError], +              title: "Unprocessable Entity", +              example: %{ +                "error" => "Someone else's status cannot be pinned" +              } +            } +          )        }      }    end @@ -197,7 +224,22 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do        parameters: [id_param()],        responses: %{          200 => status_response(), -        400 => Operation.response("Error", "application/json", ApiError) +        400 => +          Operation.response("Bad Request", "application/json", %Schema{ +            allOf: [ApiError], +            title: "Unprocessable Entity", +            example: %{ +              "error" => "You have already pinned the maximum number of statuses" +            } +          }), +        404 => +          Operation.response("Not found", "application/json", %Schema{ +            allOf: [ApiError], +            title: "Unprocessable Entity", +            example: %{ +              "error" => "Record not found" +            } +          })        }      }    end diff --git a/lib/pleroma/web/api_spec/schemas/boolean_like.ex b/lib/pleroma/web/api_spec/schemas/boolean_like.ex index eb001c5bb..778158f66 100644 --- a/lib/pleroma/web/api_spec/schemas/boolean_like.ex +++ b/lib/pleroma/web/api_spec/schemas/boolean_like.ex @@ -3,6 +3,7 @@  # SPDX-License-Identifier: AGPL-3.0-only  defmodule Pleroma.Web.ApiSpec.Schemas.BooleanLike do +  alias OpenApiSpex.Cast    alias OpenApiSpex.Schema    require OpenApiSpex @@ -27,10 +28,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.BooleanLike do        %Schema{type: :boolean},        %Schema{type: :string},        %Schema{type: :integer} -    ] +    ], +    "x-validate": __MODULE__    }) -  def after_cast(value, _schmea) do -    {:ok, Pleroma.Web.ControllerHelper.truthy_param?(value)} +  def cast(%Cast{value: value} = context) do +    context +    |> Map.put(:value, Pleroma.Web.ControllerHelper.truthy_param?(value)) +    |> Cast.ok()    end  end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 42fa98718..3d042dc19 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -194,6 +194,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do            parent_visible: %Schema{              type: :boolean,              description: "`true` if the parent post is visible to the user" +          }, +          pinned_at: %Schema{ +            type: :string, +            format: "date-time", +            nullable: true, +            description: +              "A datetime (ISO 8601) that states when the post was pinned or `null` if the post is not pinned"            }          }        }, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b003e30c7..1b5f8491e 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -228,17 +228,7 @@ defmodule Pleroma.Web.CommonAPI do        {:find_object, _} ->          {:error, :not_found} -      {:common_pipeline, -       { -         :error, -         { -           :validate_object, -           { -             :error, -             changeset -           } -         } -       }} = e -> +      {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e ->          if {:object, {"already liked by this actor", []}} in changeset.errors do            {:ok, :already_liked}          else @@ -411,29 +401,58 @@ defmodule Pleroma.Web.CommonAPI do      end    end -  def pin(id, %{ap_id: user_ap_id} = user) do -    with %Activity{ -           actor: ^user_ap_id, -           data: %{"type" => "Create"}, -           object: %Object{data: %{"type" => object_type}} -         } = activity <- Activity.get_by_id_with_object(id), -         true <- object_type in ["Note", "Article", "Question"], -         true <- Visibility.is_public?(activity), -         {:ok, _user} <- User.add_pinnned_activity(user, activity) do +  @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} +  def pin(id, %User{} = user) do +    with %Activity{} = activity <- create_activity_by_id(id), +         true <- activity_belongs_to_actor(activity, user.ap_id), +         true <- object_type_is_allowed_for_pin(activity.object), +         true <- activity_is_public(activity), +         {:ok, pin_data, _} <- Builder.pin(user, activity.object), +         {:ok, _pin, _} <- +           Pipeline.common_pipeline(pin_data, +             local: true, +             activity_id: id +           ) do        {:ok, activity}      else -      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} -      _ -> {:error, dgettext("errors", "Could not pin")} +      {:error, {:side_effects, error}} -> error +      error -> error      end    end +  defp create_activity_by_id(id) do +    with nil <- Activity.create_by_id_with_object(id) do +      {:error, :not_found} +    end +  end + +  defp activity_belongs_to_actor(%{actor: actor}, actor), do: true +  defp activity_belongs_to_actor(_, _), do: {:error, :ownership_error} + +  defp object_type_is_allowed_for_pin(%{data: %{"type" => type}}) do +    with false <- type in ["Note", "Article", "Question"] do +      {:error, :not_allowed} +    end +  end + +  defp activity_is_public(activity) do +    with false <- Visibility.is_public?(activity) do +      {:error, :visibility_error} +    end +  end + +  @spec unpin(String.t(), User.t()) :: {:ok, User.t()} | {:error, term()}    def unpin(id, user) do -    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id), -         {:ok, _user} <- User.remove_pinnned_activity(user, activity) do +    with %Activity{} = activity <- create_activity_by_id(id), +         {:ok, unpin_data, _} <- Builder.unpin(user, activity.object), +         {:ok, _unpin, _} <- +           Pipeline.common_pipeline(unpin_data, +             local: true, +             activity_id: activity.id, +             expires_at: activity.data["expires_at"], +             featured_address: user.featured_address +           ) do        {:ok, activity} -    else -      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} -      _ -> {:error, dgettext("errors", "Could not unpin")}      end    end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 73f1b0931..80a9fa7bb 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -5,6 +5,7 @@  defmodule Pleroma.Web.CommonAPI.ActivityDraft do    alias Pleroma.Activity    alias Pleroma.Conversation.Participation +  alias Pleroma.Object    alias Pleroma.Web.CommonAPI    alias Pleroma.Web.CommonAPI.Utils @@ -179,13 +180,39 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do    end    defp sensitive(draft) do -    sensitive = draft.params[:sensitive] || Enum.member?(draft.tags, {"#nsfw", "nsfw"}) +    sensitive = draft.params[:sensitive]      %__MODULE__{draft | sensitive: sensitive}    end    defp object(draft) do      emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji) +    # Sometimes people create posts with subject containing emoji, +    # since subjects are usually copied this will result in a broken +    # subject when someone replies from an instance that does not have +    # the emoji or has it under different shortcode. This is an attempt +    # to mitigate this by copying emoji from inReplyTo if they are present +    # in the subject. +    summary_emoji = +      with %Activity{} <- draft.in_reply_to, +           %Object{data: %{"tag" => [_ | _] = tag}} <- Object.normalize(draft.in_reply_to) do +        Enum.reduce(tag, %{}, fn +          %{"type" => "Emoji", "name" => name, "icon" => %{"url" => url}}, acc -> +            if String.contains?(draft.summary, name) do +              Map.put(acc, name, url) +            else +              acc +            end + +          _, acc -> +            acc +        end) +      else +        _ -> %{} +      end + +    emoji = Map.merge(emoji, summary_emoji) +      object =        Utils.make_note_data(draft)        |> Map.put("emoji", emoji) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 9587dfa25..94a378e11 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -217,7 +217,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do      draft.status      |> format_input(content_type, options)      |> maybe_add_attachments(draft.attachments, attachment_links) -    |> maybe_add_nsfw_tag(draft.params)    end    defp get_content_type(content_type) do @@ -228,13 +227,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do      end    end -  defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive}) -       when sensitive in [true, "True", "true", "1"] do -    {text, mentions, [{"#nsfw", "nsfw"} | tags]} -  end - -  defp maybe_add_nsfw_tag(data, _), do: data -    def make_context(_, %Participation{} = participation) do      Repo.preload(participation, :conversation).conversation.ap_id    end @@ -294,7 +286,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do    def format_input(text, "text/markdown", options) do      text      |> Formatter.mentions_escape(options) -    |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer}) +    |> Formatter.markdown_to_html()      |> Formatter.linkify(options)      |> Formatter.html_escape("text/html")    end diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index df97d2f46..66940f311 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -32,6 +32,7 @@ defmodule Pleroma.Web.Feed.FeedView do      %{        activity: activity, +      object: object,        data: Map.get(object, :data),        actor: actor      } diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 58d35da1e..fa7879caf 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.Feed.UserController do    def feed_redirect(conn, %{"nickname" => nickname}) do      with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do -      redirect(conn, external: "#{user_feed_url(conn, :feed, user.nickname)}.atom") +      redirect(conn, external: "#{Routes.user_feed_url(conn, :feed, user.nickname)}.atom")      end    end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index eb6639fc5..4920d65da 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do    defp redirect_to_oauth_form(conn, _params) do      with {:ok, app} <- local_mastofe_app() do        path = -        o_auth_path(conn, :authorize, +        Routes.o_auth_path(conn, :authorize,            response_type: "code",            client_id: app.client_id,            redirect_uri: ".", @@ -90,7 +90,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do    defp local_mastodon_post_login_path(conn) do      case get_session(conn, :return_to) do        nil -> -        masto_fe_path(conn, :index, ["getting-started"]) +        Routes.masto_fe_path(conn, :index, ["getting-started"])        return_to ->          delete_session(conn, :return_to) diff --git a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex index d25f84837..84621500e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex @@ -30,6 +30,12 @@ defmodule Pleroma.Web.MastodonAPI.FallbackController do      |> json(%{error: error_message})    end +  def call(conn, {:error, status, message}) do +    conn +    |> put_status(status) +    |> json(%{error: message}) +  end +    def call(conn, _) do      conn      |> put_status(:internal_server_error) diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 267d0f03b..c7a5267d4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -5,7 +5,7 @@  defmodule Pleroma.Web.MastodonAPI.InstanceController do    use Pleroma.Web, :controller -  plug(OpenApiSpex.Plug.CastAndValidate) +  plug(Pleroma.Web.ApiSpec.CastAndValidate)    plug(      :skip_plug, diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index d1a58d5e1..724dc5c5d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do    alias Pleroma.Web.CommonAPI    alias Pleroma.Web.MastodonAPI.AccountView    alias Pleroma.Web.MastodonAPI.ScheduledActivityView -  # alias Pleroma.Web.OAuth.Token +  alias Pleroma.Web.OAuth.Token    alias Pleroma.Web.Plugs.OAuthScopesPlug    alias Pleroma.Web.Plugs.RateLimiter @@ -260,6 +260,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do    def pin(%{assigns: %{user: user}} = conn, %{id: ap_id_or_id}) do      with {:ok, activity} <- CommonAPI.pin(ap_id_or_id, user) do        try_render(conn, "show.json", activity: activity, for: user, as: :activity) +    else +      {:error, :pinned_statuses_limit_reached} -> +        {:error, "You have already pinned the maximum number of statuses"} + +      {:error, :ownership_error} -> +        {:error, :unprocessable_entity, "Someone else's status cannot be pinned"} + +      {:error, :visibility_error} -> +        {:error, :unprocessable_entity, "Non-public status cannot be pinned"} + +      error -> +        error      end    end @@ -420,16 +432,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do      )    end -  # Deactivated for 2.3.0 -  # defp put_application(params, -  #   %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do -  #   if user.disclose_client do -  #     %{client_name: client_name, website: website} = Repo.preload(token, :app).app -  #     Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) -  #   else -  #     Map.put(params, :generator, nil) -  #   end -  # end +  defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do +    if user.disclose_client do +      %{client_name: client_name, website: website} = Repo.preload(token, :app).app +      Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) +    else +      Map.put(params, :generator, nil) +    end +  end    defp put_application(params, _), do: Map.put(params, :generator, nil)  end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index cef299aa4..c611958be 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -133,34 +133,25 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do    end    defp hashtag_fetching(params, user, local_only) do -    tags = +    # Note: not sanitizing tag options at this stage (may be mix-cased, have duplicates etc.) +    tags_any =        [params[:tag], params[:any]]        |> List.flatten() -      |> Enum.uniq() -      |> Enum.reject(&is_nil/1) -      |> Enum.map(&String.downcase/1) - -    tag_all = -      params -      |> Map.get(:all, []) -      |> Enum.map(&String.downcase/1) - -    tag_reject = -      params -      |> Map.get(:none, []) -      |> Enum.map(&String.downcase/1) - -    _activities = -      params -      |> Map.put(:type, "Create") -      |> Map.put(:local_only, local_only) -      |> Map.put(:blocking_user, user) -      |> Map.put(:muting_user, user) -      |> Map.put(:user, user) -      |> Map.put(:tag, tags) -      |> Map.put(:tag_all, tag_all) -      |> Map.put(:tag_reject, tag_reject) -      |> ActivityPub.fetch_public_activities() +      |> Enum.filter(& &1) + +    tag_all = Map.get(params, :all, []) +    tag_reject = Map.get(params, :none, []) + +    params +    |> Map.put(:type, "Create") +    |> Map.put(:local_only, local_only) +    |> Map.put(:blocking_user, user) +    |> Map.put(:muting_user, user) +    |> Map.put(:user, user) +    |> Map.put(:tag, tags_any) +    |> Map.put(:tag_all, tag_all) +    |> Map.put(:tag_reject, tag_reject) +    |> ActivityPub.fetch_public_activities()    end    # GET /api/v1/timelines/tag/:tag diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index ac25aefdd..9e9de33f6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -292,6 +292,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do      |> maybe_put_allow_following_move(user, opts[:for])      |> maybe_put_unread_conversation_count(user, opts[:for])      |> maybe_put_unread_notification_count(user, opts[:for]) +    |> maybe_put_email_address(user, opts[:for])    end    defp username_from_nickname(string) when is_binary(string) do @@ -403,6 +404,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do    defp maybe_put_unread_notification_count(data, _, _), do: data +  defp maybe_put_email_address(data, %User{id: user_id}, %User{id: user_id} = user) do +    Kernel.put_in( +      data, +      [:pleroma, :email], +      user.email +    ) +  end + +  defp maybe_put_email_address(data, _, _), do: data +    defp image_url(%{"url" => [%{"href" => href} | _]}), do: href    defp image_url(_), do: nil  end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 73205fb6d..dac68d8e6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -23,7 +23,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do          streaming_api: Pleroma.Web.Endpoint.websocket_url()        },        stats: Pleroma.Stats.get_stats(), -      thumbnail: Pleroma.Web.base_url() <> Keyword.get(instance, :instance_thumbnail), +      thumbnail: +        URI.merge(Pleroma.Web.base_url(), Keyword.get(instance, :instance_thumbnail)) |> to_string,        languages: ["en"],        registrations: Keyword.get(instance, :registrations_open),        approval_required: Keyword.get(instance, :account_approval_required), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bac897a57..8fdf30883 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do    alias Pleroma.Activity    alias Pleroma.HTML +  alias Pleroma.Maps    alias Pleroma.Object    alias Pleroma.Repo    alias Pleroma.User @@ -124,16 +125,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do        ) do      user = CommonAPI.get_user(activity.data["actor"])      created_at = Utils.to_masto_date(activity.data["published"]) -    activity_object = Object.normalize(activity, fetch: false) +    object = Object.normalize(activity, fetch: false)      reblogged_parent_activity =        if opts[:parent_activities] do          Activity.Queries.find_by_object_ap_id(            opts[:parent_activities], -          activity_object.data["id"] +          object.data["id"]          )        else -        Activity.create_by_object_ap_id(activity_object.data["id"]) +        Activity.create_by_object_ap_id(object.data["id"])          |> Activity.with_preloaded_bookmark(opts[:for])          |> Activity.with_set_thread_muted_field(opts[:for])          |> Repo.one() @@ -142,7 +143,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do      reblog_rendering_opts = Map.put(opts, :activity, reblogged_parent_activity)      reblogged = render("show.json", reblog_rendering_opts) -    favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || []) +    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])      bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil @@ -152,10 +153,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do        |> Enum.filter(& &1)        |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) +    {pinned?, pinned_at} = pin_data(object, user) +      %{        id: to_string(activity.id), -      uri: activity_object.data["id"], -      url: activity_object.data["id"], +      uri: object.data["id"], +      url: object.data["id"],        account:          AccountView.render("show.json", %{            user: user, @@ -173,18 +176,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do        favourited: present?(favorited),        bookmarked: present?(bookmarked),        muted: false, -      pinned: pinned?(activity, user), +      pinned: pinned?,        sensitive: false,        spoiler_text: "",        visibility: get_visibility(activity),        media_attachments: reblogged[:media_attachments] || [],        mentions: mentions,        tags: reblogged[:tags] || [], -      application: build_application(activity_object.data["generator"]), +      application: build_application(object.data["generator"]),        language: nil,        emojis: [],        pleroma: %{ -        local: activity.local +        local: activity.local, +        pinned_at: pinned_at        }      }    end @@ -198,8 +202,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do      like_count = object.data["like_count"] || 0      announcement_count = object.data["announcement_count"] || 0 -    tags = object.data["tag"] || [] -    sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") +    hashtags = Object.hashtags(object) +    sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + +    tags = Object.tags(object)      tag_mentions =        tags @@ -314,6 +320,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do            fn for_user, user -> User.mutes?(for_user, user) end          ) +    {pinned?, pinned_at} = pin_data(object, user) +      %{        id: to_string(activity.id),        uri: object.data["id"], @@ -337,7 +345,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do        favourited: present?(favorited),        bookmarked: present?(bookmarked),        muted: muted, -      pinned: pinned?(activity, user), +      pinned: pinned?,        sensitive: sensitive,        spoiler_text: summary,        visibility: get_visibility(object), @@ -358,7 +366,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do          direct_conversation_id: direct_conversation_id,          thread_muted: thread_muted?,          emoji_reactions: emoji_reactions, -        parent_visible: visible_for_user?(reply_to, opts[:for]) +        parent_visible: visible_for_user?(reply_to, opts[:for]), +        pinned_at: pinned_at        }      }    end @@ -379,12 +388,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do      page_url = page_url_data |> to_string -    image_url = +    image_url_data =        if is_binary(rich_media["image"]) do -        URI.merge(page_url_data, URI.parse(rich_media["image"])) -        |> to_string +        URI.parse(rich_media["image"]) +      else +        nil        end +    image_url = build_image_url(image_url_data, page_url_data) +      %{        type: "link",        provider_name: page_url_data.host, @@ -406,6 +418,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do      media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"      href = attachment_url["href"] |> MediaProxy.url()      href_preview = attachment_url["href"] |> MediaProxy.preview_url() +    meta = render("attachment_meta.json", %{attachment: attachment})      type =        cond do @@ -428,8 +441,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do        pleroma: %{mime_type: media_type},        blurhash: attachment["blurhash"]      } +    |> Maps.put_if_present(:meta, meta) +  end + +  def render("attachment_meta.json", %{ +        attachment: %{"url" => [%{"width" => width, "height" => height} | _]} +      }) +      when is_integer(width) and is_integer(height) do +    %{ +      original: %{ +        width: width, +        height: height, +        aspect: width / height +      } +    }    end +  def render("attachment_meta.json", _), do: nil +    def render("context.json", %{activity: activity, activities: activities, user: user}) do      %{ancestors: ancestors, descendants: descendants} =        activities @@ -524,8 +553,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do    defp present?(false), do: false    defp present?(_), do: true -  defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}), -    do: id in pinned_activities +  defp pin_data(%Object{data: %{"id" => object_id}}, %User{pinned_objects: pinned_objects}) do +    if pinned_at = pinned_objects[object_id] do +      {true, Utils.to_masto_date(pinned_at)} +    else +      {false, nil} +    end +  end    defp build_emoji_map(emoji, users, current_user) do      %{ @@ -536,6 +570,27 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do    end    @spec build_application(map() | nil) :: map() | nil -  defp build_application(%{type: _type, name: name, url: url}), do: %{name: name, website: url} +  defp build_application(%{"type" => _type, "name" => name, "url" => url}), +    do: %{name: name, website: url} +    defp build_application(_), do: nil + +  # Workaround for Elixir issue #10771 +  # Avoid applying URI.merge unless necessary +  # TODO: revert to always attempting URI.merge(image_url_data, page_url_data) +  # when Elixir 1.12 is the minimum supported version +  @spec build_image_url(struct() | nil, struct()) :: String.t() | nil +  defp build_image_url( +         %URI{scheme: image_scheme, host: image_host} = image_url_data, +         %URI{} = _page_url_data +       ) +       when not is_nil(image_scheme) and not is_nil(image_host) do +    image_url_data |> to_string +  end + +  defp build_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do +    URI.merge(page_url_data, image_url_data) |> to_string +  end + +  defp build_image_url(_, _), do: nil  end diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index 27f337138..d0d4bb4b3 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -121,6 +121,11 @@ defmodule Pleroma.Web.MediaProxy do      end    end +  def decode_url(encoded) do +    [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") +    decode_url(sig, base64) +  end +    defp signed_url(url) do      :crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url)    end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 215d97b3a..42f4d768f 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -427,7 +427,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do        |> Map.put("state", state)      # Handing the request to Ueberauth -    redirect(conn, to: o_auth_path(conn, :request, provider, params)) +    redirect(conn, to: Routes.o_auth_path(conn, :request, provider, params))    end    def request(%Plug.Conn{} = conn, params) do @@ -601,7 +601,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do    end    # Special case: Local MastodonFE -  defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) +  defp redirect_uri(%Plug.Conn{} = conn, "."), do: Routes.auth_url(conn, :login)    defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri diff --git a/lib/pleroma/web/o_auth/o_auth_view.ex b/lib/pleroma/web/o_auth/o_auth_view.ex index 281bbcc3c..1419c96a2 100644 --- a/lib/pleroma/web/o_auth/o_auth_view.ex +++ b/lib/pleroma/web/o_auth/o_auth_view.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OAuth.OAuthView do    def render("token.json", %{token: token} = opts) do      response = %{ +      id: token.id,        token_type: "Bearer",        access_token: token.token,        refresh_token: token.refresh_token, diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 315657e9c..fc5d16771 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do    action_fallback(Pleroma.Web.MastodonAPI.FallbackController)    plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) -  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) +  plug(Pleroma.Web.ApiSpec.CastAndValidate)    defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 4adc685fe..dcd54b1af 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do      %{scopes: ["read:chats"]} when action in [:messages, :index, :index2, :show]    ) -  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) +  plug(Pleroma.Web.ApiSpec.CastAndValidate)    defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex index 6d9a11fb6..078d470d9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.PleromaAPI.UserImportController do    plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks)    plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes) -  plug(OpenApiSpex.Plug.CastAndValidate) +  plug(Pleroma.Web.ApiSpec.CastAndValidate)    defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation    def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do diff --git a/lib/pleroma/web/plugs/frontend_static.ex b/lib/pleroma/web/plugs/frontend_static.ex index eb385e94d..e7c943b41 100644 --- a/lib/pleroma/web/plugs/frontend_static.ex +++ b/lib/pleroma/web/plugs/frontend_static.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Plugs.FrontendStatic do    """    @behaviour Plug -  @api_routes Pleroma.Web.get_api_routes() +  @api_routes Pleroma.Web.Router.get_api_routes()    def file_path(path, frontend_type \\ :primary) do      if configuration = Pleroma.Config.get([:frontends, frontend_type]) do diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 0025b042a..d1e6cc9d3 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -48,7 +48,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do        {"x-content-type-options", "nosniff"},        {"referrer-policy", referrer_policy},        {"x-download-options", "noopen"}, -      {"content-security-policy", csp_string()} +      {"content-security-policy", csp_string()}, +      {"permissions-policy", "interest-cohort=()"}      ]      headers = diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 72ad14f05..95d56699e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -140,6 +140,10 @@ defmodule Pleroma.Web.Router do      plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug)    end +  pipeline :static_fe do +    plug(Pleroma.Web.Plugs.StaticFEPlug) +  end +    scope "/api/v1/pleroma", Pleroma.Web.TwitterAPI do      pipe_through(:pleroma_api) @@ -204,7 +208,7 @@ defmodule Pleroma.Web.Router do      get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials)      patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials) -    get("/users", UserController, :list) +    get("/users", UserController, :index)      get("/users/:nickname", UserController, :show)      get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses)      get("/users/:nickname/chats", AdminAPIController, :list_user_chats) @@ -631,7 +635,7 @@ defmodule Pleroma.Web.Router do    scope "/", Pleroma.Web do      # Note: html format is supported only if static FE is enabled      # Note: http signature is only considered for json requests (no auth for non-json requests) -    pipe_through([:accepts_html_json, :http_signature, Pleroma.Web.Plugs.StaticFEPlug]) +    pipe_through([:accepts_html_json, :http_signature, :static_fe])      get("/objects/:uuid", OStatus.OStatusController, :object)      get("/activities/:uuid", OStatus.OStatusController, :activity) @@ -645,7 +649,7 @@ defmodule Pleroma.Web.Router do    scope "/", Pleroma.Web do      # Note: html format is supported only if static FE is enabled      # Note: http signature is only considered for json requests (no auth for non-json requests) -    pipe_through([:accepts_html_xml_json, :http_signature, Pleroma.Web.Plugs.StaticFEPlug]) +    pipe_through([:accepts_html_xml_json, :http_signature, :static_fe])      # Note: returns user _profile_ for json requests, redirects to user _feed_ for non-json ones      get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed) @@ -653,7 +657,7 @@ defmodule Pleroma.Web.Router do    scope "/", Pleroma.Web do      # Note: html format is supported only if static FE is enabled -    pipe_through([:accepts_html_xml, Pleroma.Web.Plugs.StaticFEPlug]) +    pipe_through([:accepts_html_xml, :static_fe])      get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)    end @@ -704,6 +708,7 @@ defmodule Pleroma.Web.Router do      # The following two are S2S as well, see `ActivityPub.fetch_follow_information_for_user/1`:      get("/users/:nickname/followers", ActivityPubController, :followers)      get("/users/:nickname/following", ActivityPubController, :following) +    get("/users/:nickname/collections/featured", ActivityPubController, :pinned)    end    scope "/", Pleroma.Web.ActivityPub do @@ -764,11 +769,11 @@ defmodule Pleroma.Web.Router do      get("/embed/:id", EmbedController, :show)    end -  scope "/proxy/", Pleroma.Web.MediaProxy do -    get("/preview/:sig/:url", MediaProxyController, :preview) -    get("/preview/:sig/:url/:filename", MediaProxyController, :preview) -    get("/:sig/:url", MediaProxyController, :remote) -    get("/:sig/:url/:filename", MediaProxyController, :remote) +  scope "/proxy/", Pleroma.Web do +    get("/preview/:sig/:url", MediaProxy.MediaProxyController, :preview) +    get("/preview/:sig/:url/:filename", MediaProxy.MediaProxyController, :preview) +    get("/:sig/:url", MediaProxy.MediaProxyController, :remote) +    get("/:sig/:url/:filename", MediaProxy.MediaProxyController, :remote)    end    if Pleroma.Config.get(:env) == :dev do @@ -821,4 +826,16 @@ defmodule Pleroma.Web.Router do      options("/*path", RedirectController, :empty)    end + +  # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ +  def get_api_routes do +    __MODULE__.__routes__() +    |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) +    |> Enum.map(fn r -> +      r.path +      |> String.split("/", trim: true) +      |> List.first() +    end) +    |> Enum.uniq() +  end  end diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 3fd150c4e..6688830ba 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,7 +22,7 @@      <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>    <% end %> -  <%= for tag <- @data["tag"] || [] do %> +  <%= for tag <- Pleroma.Object.hashtags(@object) do %>      <category term="<%= tag %>"></category>    <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 947bbb099..592b9dcdc 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -22,7 +22,7 @@    <link rel="ostatus:conversation"><%= activity_context(@activity) %></link> -  <%= for tag <- @data["tag"] || [] do %> +  <%= for tag <- Pleroma.Object.hashtags(@object) do %>      <category term="<%= tag %>"></category>    <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index cf5874a91..c2de28fe4 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,7 +41,7 @@        <% end %>      <% end %> -    <%= for tag <- @data["tag"] || [] do %> +    <%= for tag <- Pleroma.Object.hashtags(@object) do %>        <category term="<%= tag %>"></category>      <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index a288539ed..de0731085 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,13 +9,13 @@        xmlns:ostatus="http://ostatus.org/schema/1.0"        xmlns:statusnet="http://status.net/schema/api/1/"> -    <id><%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %></id> +    <id><%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %></id>      <title>#<%= @tag %></title>      <subtitle>These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse.</subtitle>      <logo><%= feed_logo() %></logo>      <updated><%= most_recent_update(@activities) %></updated> -    <link rel="self" href="<%= '#{tag_feed_url(@conn, :feed, @tag)}.atom'  %>" type="application/atom+xml"/> +    <link rel="self" href="<%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.atom'  %>" type="application/atom+xml"/>      <%= for activity <- @activities do %>      <%= render @view_module, "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %>      <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index eeda01a04..9c3613feb 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -5,7 +5,7 @@      <title>#<%= @tag %></title>      <description>These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse.</description> -    <link><%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %></link> +    <link><%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %></link>      <webfeeds:logo><%= feed_logo() %></webfeeds:logo>      <webfeeds:accentColor>2b90d9</webfeeds:accentColor>      <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c6acd848f..5c1f0ecbc 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -6,16 +6,16 @@    xmlns:poco="http://portablecontacts.net/spec/1.0"    xmlns:ostatus="http://ostatus.org/schema/1.0"> -  <id><%= user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %></id> +  <id><%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %></id>    <title><%= @user.nickname <> "'s timeline" %></title>    <updated><%= most_recent_update(@activities, @user) %></updated>    <logo><%= logo(@user) %></logo> -  <link rel="self" href="<%= '#{user_feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/> +  <link rel="self" href="<%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/>    <%= render @view_module, "_author.atom", assigns %>    <%= if last_activity(@activities) do %> -    <link rel="next" href="<%= '#{user_feed_url(@conn, :feed, @user.nickname)}.atom?max_id=#{last_activity(@activities).id}' %>" type="application/atom+xml"/> +    <link rel="next" href="<%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom?max_id=#{last_activity(@activities).id}' %>" type="application/atom+xml"/>    <% end %>    <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index d69120480..6b842a085 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,16 +1,16 @@  <?xml version="1.0" encoding="UTF-8" ?>  <rss version="2.0">    <channel> -    <guid><%= user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %></guid> +    <guid><%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %></guid>      <title><%= @user.nickname <> "'s timeline" %></title>      <updated><%= most_recent_update(@activities, @user) %></updated>      <image><%= logo(@user) %></image> -    <link><%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss' %></link> +    <link><%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss' %></link>      <%= render @view_module, "_author.rss", assigns %>      <%= if last_activity(@activities) do %> -      <link rel="next"><%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %></link> +      <link rel="next"><%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %></link>      <% end %>      <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/index.html.eex index c330960fa..6f2b98957 100644 --- a/lib/pleroma/web/templates/masto_fe/index.html.eex +++ b/lib/pleroma/web/templates/masto_fe/index.html.eex @@ -7,7 +7,7 @@  <%= Config.get([:instance, :name]) %>  </title>  <link rel="icon" type="image/png" href="/favicon.png"/> -<link rel="manifest" type="applicaton/manifest+json" href="<%= masto_fe_path(Pleroma.Web.Endpoint, :manifest) %>" /> +<link rel="manifest" type="applicaton/manifest+json" href="<%= Routes.masto_fe_path(Pleroma.Web.Endpoint, :manifest) %>" />  <meta name="theme-color" content="<%= Config.get([:manifest, :theme_color]) %>" /> diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 5ab59b57b..b9daa8d8b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -7,7 +7,7 @@  <h2>Two-factor recovery</h2> -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>  <div class="input">    <%= label f, :code, "Recovery code" %>    <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> @@ -19,6 +19,6 @@  <%= submit "Verify" %>  <% end %> -<a href="<%= mfa_path(@conn, :show, %{challenge_type: "totp", mfa_token: @mfa_token, state: @state, redirect_uri: @redirect_uri}) %>"> +<a href="<%= Routes.mfa_path(@conn, :show, %{challenge_type: "totp", mfa_token: @mfa_token, state: @state, redirect_uri: @redirect_uri}) %>">    Enter a two-factor code  </a> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af85777eb..29ea7c5fb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -7,7 +7,7 @@  <h2>Two-factor authentication</h2> -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>  <div class="input">    <%= label f, :code, "Authentication code" %>    <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> @@ -19,6 +19,6 @@  <%= submit "Verify" %>  <% end %> -<a href="<%= mfa_path(@conn, :show, %{challenge_type: "recovery", mfa_token: @mfa_token, state: @state, redirect_uri: @redirect_uri}) %>"> +<a href="<%= Routes.mfa_path(@conn, :show, %{challenge_type: "recovery", mfa_token: @mfa_token, state: @state, redirect_uri: @redirect_uri}) %>">    Enter a two-factor recovery code  </a> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 4a0718851..dc4521a62 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,6 +1,6 @@  <h2>Sign in with external provider</h2> -<%= form_for @conn, o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %>    <div style="display: none">      <%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %>    </div> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index facedc8db..99f900fb7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -8,7 +8,7 @@  <h2>Registration Details</h2>  <p>If you'd like to register a new account, please provide the details below.</p> -<%= form_for @conn, o_auth_path(@conn, :register), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :register), [as: "authorization"], fn f -> %>  <div class="input">    <%= label f, :nickname, "Nickname" %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 1a85818ec..2846ec7e7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -5,7 +5,7 @@  <p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p>  <% end %> -<%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %>  <%= if @user do %>    <div class="account-header"> diff --git a/lib/pleroma/web/templates/twitter_api/password/reset.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex index 7d3ef6b0d..fbcacdc14 100644 --- a/lib/pleroma/web/templates/twitter_api/password/reset.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex @@ -1,5 +1,5 @@  <h2>Password Reset for <%= @user.nickname %></h2> -<%= form_for @conn, reset_password_path(@conn, :do_reset), [as: "data"], fn f -> %> +<%= form_for @conn, Routes.reset_password_path(@conn, :do_reset), [as: "data"], fn f -> %>    <div class="form-row">      <%= label f, :password, "Password" %>      <%= password_input f, :password %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex index 5ba192cd7..a7be53091 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex @@ -4,7 +4,7 @@      <h2>Remote follow</h2>      <img height="128" width="128" src="<%= avatar_url(@followee) %>">      <p><%= @followee.nickname %></p> -    <%= form_for @conn, remote_follow_path(@conn, :do_follow), [as: "user"], fn f -> %> +    <%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "user"], fn f -> %>      <%= hidden_input f, :id, value: @followee.id %>      <%= submit "Authorize" %>      <% end %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex index df44988ee..a8026fa9d 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex @@ -4,7 +4,7 @@  <h2>Log in to follow</h2>  <p><%= @followee.nickname %></p>  <img height="128" width="128" src="<%= avatar_url(@followee) %>"> -<%= form_for @conn, remote_follow_path(@conn, :do_follow), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "authorization"], fn f -> %>  <%= text_input f, :name, placeholder: "Username", required: true %>  <br>  <%= password_input f, :password, placeholder: "Password", required: true %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex index adc3a3e3d..a54ed83b5 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex @@ -4,7 +4,7 @@  <h2>Two-factor authentication</h2>  <p><%= @followee.nickname %></p>  <img height="128" width="128" src="<%= avatar_url(@followee) %>"> -<%= form_for @conn, remote_follow_path(@conn, :do_follow), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "mfa"], fn f -> %>  <%= text_input f, :code, placeholder: "Authentication code", required: true %>  <br>  <%= hidden_input f, :id, value: @followee.id %> diff --git a/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex index f60accebf..a6b313d8a 100644 --- a/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex +++ b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex @@ -2,7 +2,7 @@    <h2>Error: <%= @error %></h2>  <% else %>    <h2>Remotely follow <%= @nickname %></h2> -  <%= form_for @conn, util_path(@conn, :remote_subscribe), [as: "user"], fn f -> %> +  <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "user"], fn f -> %>    <%= hidden_input f, :nickname, value: @nickname %>    <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %>    <%= submit "Follow" %> diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 6ca02fbd7..9843cc362 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do    defp follow_status(conn, _user, acct) do      with {:ok, object} <- Fetcher.fetch_object_from_id(acct),           %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object.data["id"]) do -      redirect(conn, to: o_status_path(conn, :notice, activity_id)) +      redirect(conn, to: Routes.o_status_path(conn, :notice, activity_id))      else        error ->          handle_follow_error(conn, error) diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index b9055cb7f..82b301949 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -79,7 +79,7 @@ defmodule Pleroma.Web.MastoFEView do        background_color: Config.get([:manifest, :background_color]),        display: "standalone",        scope: Pleroma.Web.base_url(), -      start_url: masto_fe_path(Pleroma.Web.Endpoint, :index, ["getting-started"]), +      start_url: Routes.masto_fe_path(Pleroma.Web.Endpoint, :index, ["getting-started"]),        categories: [          "social"        ], diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 15002b29f..21b10e654 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -94,52 +94,56 @@ defmodule Pleroma.Web.WebFinger do      |> XmlBuilder.to_doc()    end -  defp webfinger_from_xml(doc) do -    subject = XML.string_from_xpath("//Subject", doc) - -    subscribe_address = -      ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template} -      |> XML.string_from_xpath(doc) - -    ap_id = -      ~s{//Link[@rel="self" and @type="application/activity+json"]/@href} -      |> XML.string_from_xpath(doc) - -    data = %{ -      "subject" => subject, -      "subscribe_address" => subscribe_address, -      "ap_id" => ap_id -    } +  defp webfinger_from_xml(body) do +    with {:ok, doc} <- XML.parse_document(body) do +      subject = XML.string_from_xpath("//Subject", doc) + +      subscribe_address = +        ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template} +        |> XML.string_from_xpath(doc) + +      ap_id = +        ~s{//Link[@rel="self" and @type="application/activity+json"]/@href} +        |> XML.string_from_xpath(doc) + +      data = %{ +        "subject" => subject, +        "subscribe_address" => subscribe_address, +        "ap_id" => ap_id +      } -    {:ok, data} +      {:ok, data} +    end    end -  defp webfinger_from_json(doc) do -    data = -      Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data -> -        case {link["type"], link["rel"]} do -          {"application/activity+json", "self"} -> -            Map.put(data, "ap_id", link["href"]) +  defp webfinger_from_json(body) do +    with {:ok, doc} <- Jason.decode(body) do +      data = +        Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data -> +          case {link["type"], link["rel"]} do +            {"application/activity+json", "self"} -> +              Map.put(data, "ap_id", link["href"]) -          {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} -> -            Map.put(data, "ap_id", link["href"]) +            {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} -> +              Map.put(data, "ap_id", link["href"]) -          {nil, "http://ostatus.org/schema/1.0/subscribe"} -> -            Map.put(data, "subscribe_address", link["template"]) +            {nil, "http://ostatus.org/schema/1.0/subscribe"} -> +              Map.put(data, "subscribe_address", link["template"]) -          _ -> -            Logger.debug("Unhandled type: #{inspect(link["type"])}") -            data -        end -      end) +            _ -> +              Logger.debug("Unhandled type: #{inspect(link["type"])}") +              data +          end +        end) -    {:ok, data} +      {:ok, data} +    end    end    def get_template_from_xml(body) do      xpath = "//Link[@rel='lrdd']/@template" -    with doc when doc != :error <- XML.parse_document(body), +    with {:ok, doc} <- XML.parse_document(body),           template when template != nil <- XML.string_from_xpath(xpath, doc) do        {:ok, template}      end @@ -192,15 +196,23 @@ defmodule Pleroma.Web.WebFinger do               address,               [{"accept", "application/xrd+xml,application/jrd+json"}]             ), -         {:ok, %{status: status, body: body}} when status in 200..299 <- response do -      doc = XML.parse_document(body) - -      if doc != :error do -        webfinger_from_xml(doc) -      else -        with {:ok, doc} <- Jason.decode(body) do -          webfinger_from_json(doc) -        end +         {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- +           response do +      case List.keyfind(headers, "content-type", 0) do +        {_, content_type} -> +          case Plug.Conn.Utils.media_type(content_type) do +            {:ok, "application", subtype, _} when subtype in ~w(xrd+xml xml) -> +              webfinger_from_xml(body) + +            {:ok, "application", subtype, _} when subtype in ~w(jrd+json json) -> +              webfinger_from_json(body) + +            _ -> +              {:error, {:content_type, content_type}} +          end + +        _ -> +          {:error, {:content_type, nil}}        end      else        e -> diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex index 2b34611ac..0ab6e9d32 100644 --- a/lib/pleroma/web/xml.ex +++ b/lib/pleroma/web/xml.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.XML do          |> :binary.bin_to_list()          |> :xmerl_scan.string(quiet: true) -      doc +      {:ok, doc}      rescue        _e ->          Logger.debug("Couldn't parse XML: #{inspect(text)}")  | 
