diff options
Diffstat (limited to 'lib/mix')
-rw-r--r-- | lib/mix/pleroma.ex | 48 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/benchmark.ex | 11 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/config.ex | 19 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/database.ex | 39 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/ecto/migrate.ex | 4 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/ecto/rollback.ex | 4 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/emoji.ex | 22 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/frontend.ex | 141 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/instance.ex | 4 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/notification_settings.ex | 18 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/refresh_counter_cache.ex | 49 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/relay.ex | 10 | ||||
-rw-r--r-- | lib/mix/tasks/pleroma/user.ex | 4 |
13 files changed, 310 insertions, 63 deletions
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..49ba2aae4 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -3,15 +3,54 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Pleroma do + @apps [ + :restarter, + :ecto, + :ecto_sql, + :postgrex, + :db_connection, + :cachex, + :flake_id, + :swoosh, + :timex + ] + @cachex_children ["object", "user", "scrubber"] @doc "Common functions to be reused in mix tasks" def start_pleroma do + Pleroma.Config.Holder.save_default() + Pleroma.Config.Oban.warn() Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do Application.put_env(:logger, :console, level: :debug) end - {:ok, _} = Application.ensure_all_started(:pleroma) + adapter = Application.get_env(:tesla, :adapter) + + apps = + if adapter == Tesla.Adapter.Gun do + [:gun | @apps] + else + [:hackney | @apps] + end + + Enum.each(apps, &Application.ensure_all_started/1) + + children = + [ + Pleroma.Repo, + {Pleroma.Config.TransferTask, false}, + Pleroma.Web.Endpoint, + {Oban, Pleroma.Config.get(Oban)} + ] ++ + http_children(adapter) + + cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) + + Supervisor.start_link(children ++ cachex_children, + strategy: :one_for_one, + name: Pleroma.Supervisor + ) if Pleroma.Config.get(:env) not in [:test, :benchmark] do pleroma_rebooted?() @@ -82,4 +121,11 @@ defmodule Mix.Pleroma do def escape_sh_path(path) do ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(') end + + defp http_children(Tesla.Adapter.Gun) do + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] + end + + defp http_children(_), do: [] end diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index dd2b9c8f2..a607d5d4f 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do "Without conn and without pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool, receive_conn: false] + pool: :no_pool, + receive_conn: false ) end, "Without conn and with pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [receive_conn: false] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) end, "With reused conn and without pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) end, "With reused conn and with pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 5c9ef6904..904c5a74b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -52,6 +52,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp do_migrate_to_db(config_file) do if File.exists?(config_file) do + shell_info("Migrating settings from file: #{Path.expand(config_file)}") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;") Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;") @@ -72,8 +73,7 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - key = inspect(key) - {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value}) + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) shell_info("Settings for key #{key} migrated.") end) @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do if Pleroma.Config.get([:configurable_from_database]) do - env = opts[:env] || "prod" + env = opts[:env] || Pleroma.Config.get(:env) config_path = if Pleroma.Config.get(:release) do @@ -105,6 +105,10 @@ defmodule Mix.Tasks.Pleroma.Config do :ok = File.close(file) System.cmd("mix", ["format", config_path]) + + shell_info( + "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs" + ) else migration_error() end @@ -112,7 +116,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migration_error do shell_error( - "Migration is not allowed in config. You can change this behavior by setting `configurable_from_database` to true." + "Migration is not allowed in config. You can change this behavior by setting `config :pleroma, configurable_from_database: true`" ) end @@ -131,12 +135,9 @@ defmodule Mix.Tasks.Pleroma.Config do end defp write(config, file) do - value = - config.value - |> ConfigDB.from_binary() - |> inspect(limit: :infinity) + value = inspect(config.value, limit: :infinity) - IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n") + IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n") config end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 82e2abdcb..7f1108dcf 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.User require Logger require Pleroma.Constants + import Ecto.Query import Mix.Pleroma use Mix.Task @@ -53,8 +54,6 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["prune_objects" | args]) do - import Ecto.Query - {options, [], []} = OptionParser.parse( args, @@ -94,8 +93,6 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["fix_likes_collections"]) do - import Ecto.Query - start_pleroma() from(object in Object, @@ -130,4 +127,38 @@ defmodule Mix.Tasks.Pleroma.Database do Maintenance.vacuum(args) end + + def run(["ensure_expiration"]) do + start_pleroma() + days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) + + Pleroma.Activity + |> join(:inner, [a], o in Object, + on: + fragment( + "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + o.data, + a.data, + a.data + ) + ) + |> where(local: true) + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([_a, o], fragment("?->>'type' = 'Note'", o.data)) + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn activities -> + Enum.each(activities, fn activity -> + expires_at = + activity.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: days) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at + }) + end) + end) + |> Stream.run() + end end diff --git a/lib/mix/tasks/pleroma/ecto/migrate.ex b/lib/mix/tasks/pleroma/ecto/migrate.ex index bc8ed29fb..e903bd171 100644 --- a/lib/mix/tasks/pleroma/ecto/migrate.ex +++ b/lib/mix/tasks/pleroma/ecto/migrate.ex @@ -41,6 +41,10 @@ defmodule Mix.Tasks.Pleroma.Ecto.Migrate do load_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do + Application.ensure_all_started(:ssl) + end + opts = if opts[:to] || opts[:step] || opts[:all], do: opts, diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index f43bd0b98..3dba952cb 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -40,6 +40,10 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do load_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + if Application.get_env(:pleroma, Pleroma.Repo)[:ssl] do + Application.ensure_all_started(:ssl) + end + opts = if opts[:to] || opts[:step] || opts[:all], do: opts, diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index f4eaeac98..1750373f9 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -15,7 +15,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, [], []} = parse_global_opts(args) url_or_path = options[:manifest] || default_manifest() - manifest = fetch_and_decode(url_or_path) + manifest = fetch_and_decode!(url_or_path) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do url_or_path = options[:manifest] || default_manifest() - manifest = fetch_and_decode(url_or_path) + manifest = fetch_and_decode!(url_or_path) for pack_name <- pack_names do if Map.has_key?(manifest, pack_name) do @@ -92,7 +92,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do ]) ) - files = fetch_and_decode(files_loc) + files = fetch_and_decode!(files_loc) IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) @@ -183,7 +183,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("Downloading the pack and generating SHA256") - binary_archive = Tesla.get!(client(), src).body + {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src) archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() IO.puts("SHA256 is #{archive_sha}") @@ -243,14 +243,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("Emoji packs have been reloaded.") end - defp fetch_and_decode(from) do + defp fetch_and_decode!(from) do with {:ok, json} <- fetch(from) do Jason.decode!(json) + else + {:error, error} -> raise "#{from} cannot be fetched. Error: #{error} occur." end end defp fetch("http" <> _ = from) do - with {:ok, %{body: body}} <- Tesla.get(client(), from) do + with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do {:ok, body} end end @@ -269,13 +271,5 @@ defmodule Mix.Tasks.Pleroma.Emoji do ) end - defp client do - middleware = [ - {Tesla.Middleware.FollowRedirects, [max_redirects: 3]} - ] - - Tesla.client(middleware) - end - defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest]) end diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex new file mode 100644 index 000000000..cbce81ab9 --- /dev/null +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -0,0 +1,141 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Frontend do + use Mix.Task + + import Mix.Pleroma + + @shortdoc "Manages bundled Pleroma frontends" + + @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md") + + def run(["install", "none" | _args]) do + shell_info("Skipping frontend installation because none was requested") + "none" + end + + def run(["install", frontend | args]) do + log_level = Logger.level() + Logger.configure(level: :warn) + start_pleroma() + + {options, [], []} = + OptionParser.parse( + args, + strict: [ + ref: :string, + static_dir: :string, + build_url: :string, + build_dir: :string, + file: :string + ] + ) + + instance_static_dir = + with nil <- options[:static_dir] do + Pleroma.Config.get!([:instance, :static_dir]) + end + + cmd_frontend_info = %{ + "name" => frontend, + "ref" => options[:ref], + "build_url" => options[:build_url], + "build_dir" => options[:build_dir] + } + + config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{}) + + frontend_info = + Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd -> + # This only overrides things that are actually set + cmd || config + end) + + ref = frontend_info["ref"] + + unless ref do + raise "No ref given or configured" + end + + dest = + Path.join([ + instance_static_dir, + "frontends", + frontend, + ref + ]) + + fe_label = "#{frontend} (#{ref})" + + tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"]) + + with {_, :ok} <- + {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])}, + shell_info("Installing #{fe_label} to #{dest}"), + :ok <- install_frontend(frontend_info, tmp_dir, dest) do + File.rm_rf!(tmp_dir) + shell_info("Frontend #{fe_label} installed to #{dest}") + + Logger.configure(level: log_level) + else + {:download_or_unzip, _} -> + shell_info("Could not download or unzip the frontend") + + _e -> + shell_info("Could not install the frontend") + end + end + + defp download_or_unzip(frontend_info, temp_dir, file) do + if file do + with {:ok, zip} <- File.read(Path.expand(file)) do + unzip(zip, temp_dir) + end + else + download_build(frontend_info, temp_dir) + end + end + + def unzip(zip, dest) do + with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do + File.rm_rf!(dest) + File.mkdir_p!(dest) + + Enum.each(unzipped, fn {filename, data} -> + path = filename + + new_file_path = Path.join(dest, path) + + new_file_path + |> Path.dirname() + |> File.mkdir_p!() + + File.write!(new_file_path, data) + end) + + :ok + end + end + + defp download_build(frontend_info, dest) do + shell_info("Downloading pre-built bundle for #{frontend_info["name"]}") + url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) + + with {:ok, %{status: 200, body: zip_body}} <- + Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do + unzip(zip_body, dest) + else + e -> {:error, e} + end + end + + defp install_frontend(frontend_info, source, dest) do + from = frontend_info["build_dir"] || "dist" + File.rm_rf!(dest) + File.mkdir_p!(dest) + File.cp_r!(Path.join([source, from]), dest) + :ok + end +end diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 86409738a..91440b453 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", - Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) + Config.get([Pleroma.Uploaders.Local, :uploads]) ) |> Path.expand() @@ -154,7 +154,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :static_dir, "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?", - Pleroma.Config.get([:instance, :static_dir]) + Config.get([:instance, :static_dir]) ) |> Path.expand() diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex index 7d65f0587..00f5ba7bf 100644 --- a/lib/mix/tasks/pleroma/notification_settings.ex +++ b/lib/mix/tasks/pleroma/notification_settings.ex @@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do @moduledoc """ Example: - > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588" # set false only for parallel588 user - > mix pleroma.notification_settings --privacy-option=true # set true for all users + > mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588" # set false only for parallel588 user + > mix pleroma.notification_settings --hide-notification-contents=true # set true for all users """ @@ -19,16 +19,16 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do OptionParser.parse( args, strict: [ - privacy_option: :boolean, + hide_notification_contents: :boolean, email_users: :string, nickname_users: :string ] ) - privacy_option = Keyword.get(options, :privacy_option) + hide_notification_contents = Keyword.get(options, :hide_notification_contents) - if not is_nil(privacy_option) do - privacy_option + if not is_nil(hide_notification_contents) do + hide_notification_contents |> build_query(options) |> Pleroma.Repo.update_all([]) end @@ -36,15 +36,15 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do shell_info("Done") end - defp build_query(privacy_option, options) do + defp build_query(hide_notification_contents, options) do query = from(u in Pleroma.User, update: [ set: [ notification_settings: fragment( - "jsonb_set(notification_settings, '{privacy_option}', ?)", - ^privacy_option + "jsonb_set(notification_settings, '{hide_notification_contents}', ?)", + ^hide_notification_contents ) ] ] diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex index 15b4dbfa6..efcbaa3b1 100644 --- a/lib/mix/tasks/pleroma/refresh_counter_cache.ex +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -17,30 +17,53 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do def run([]) do Mix.Pleroma.start_pleroma() - ["public", "unlisted", "private", "direct"] - |> Enum.each(fn visibility -> - count = status_visibility_count_query(visibility) - name = "status_visibility_#{visibility}" - CounterCache.set(name, count) - Mix.Pleroma.shell_info("Set #{name} to #{count}") + instances = + Activity + |> distinct([a], true) + |> select([a], fragment("split_part(?, '/', 3)", a.actor)) + |> Repo.all() + + instances + |> Enum.with_index(1) + |> Enum.each(fn {instance, i} -> + counters = instance_counters(instance) + CounterCache.set(instance, counters) + + Mix.Pleroma.shell_info( + "[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}" + ) end) Mix.Pleroma.shell_info("Done") end - defp status_visibility_count_query(visibility) do + defp instance_counters(instance) do + counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0} + Activity - |> where( + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance)) + |> select( + [a], + {fragment( + "activity_visibility(?, ?, ?)", + a.actor, + a.recipients, + a.data + ), count(a.id)} + ) + |> group_by( [a], fragment( - "activity_visibility(?, ?, ?) = ?", + "activity_visibility(?, ?, ?)", a.actor, a.recipients, - a.data, - ^visibility + a.data ) ) - |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30)) + |> Repo.all(timeout: :timer.minutes(30)) + |> Enum.reduce(counters, fn {visibility, count}, acc -> + Map.put(acc, visibility, count) + end) end end diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index c3312507e..a6d8d6c1c 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -35,10 +35,16 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() - with {:ok, list} <- Relay.list(true) do - list |> Enum.each(&shell_info(&1)) + with {:ok, list} <- Relay.list() do + Enum.each(list, &print_relay_url/1) else {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") end end + + defp print_relay_url(%{followed_back: false} = relay) do + shell_info("#{relay.actor} - no Accept received (relay didn't follow back)") + end + + defp print_relay_url(relay), do: shell_info(relay.actor) end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index bca7e87bf..01824aa18 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -232,7 +232,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.tag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") @@ -245,7 +245,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.untag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") |