From b3764423251c963a5ca007517189f556bfe95155 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 11 Jul 2020 10:36:36 +0300 Subject: MediaProxy whitelist setting now supports hosts with scheme added deprecation warning about using bare domains --- config/description.exs | 4 ++-- config/test.exs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'config') diff --git a/config/description.exs b/config/description.exs index b0cc8d527..432705307 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1775,8 +1775,8 @@ config :pleroma, :config_description, [ %{ key: :whitelist, type: {:list, :string}, - description: "List of domains to bypass the mediaproxy", - suggestions: ["example.com"] + description: "List of hosts with scheme to bypass the mediaproxy", + suggestions: ["http://example.com"] } ] }, diff --git a/config/test.exs b/config/test.exs index d45c36b7b..abcf793e5 100644 --- a/config/test.exs +++ b/config/test.exs @@ -113,6 +113,11 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true config :pleroma, :instances_favicons, enabled: true +config :pleroma, Pleroma.Uploaders.S3, + bucket: nil, + streaming_enabled: true, + public_endpoint: nil + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else -- cgit v1.2.3 From 9b225db7d86289fb9d9c51f62e6ec29f6c07f60d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 14 Jul 2020 11:58:41 +0300 Subject: [#1940] Applied rate limit for requests with bad `admin_token`. Added doc warnings on `admin_token` setting. --- config/description.exs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'config') diff --git a/config/description.exs b/config/description.exs index 84dcdb87e..8ec4b712f 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2008,13 +2008,15 @@ config :pleroma, :config_description, [ label: "Pleroma Admin Token", type: :group, description: - "Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the `admin_token` parameter", + "Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the `admin_token` parameter (risky; use HTTP Basic Auth or OAuth-based authentication if possible)", children: [ %{ key: :admin_token, type: :string, description: "Admin token", - suggestions: ["We recommend a secure random string or UUID"] + suggestions: [ + "We recommend NOT setting the value do to increased security risk; if set, use a secure random long string or UUID (and change it as often as possible)" + ] } ] }, -- cgit v1.2.3 From ce314e6fe236c7a41535dd8a9a0f097c74c6f1ce Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 14 Jul 2020 11:24:58 -0500 Subject: Clarify description and suggestion --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config') diff --git a/config/description.exs b/config/description.exs index 8ec4b712f..2b41e7dac 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2008,14 +2008,14 @@ config :pleroma, :config_description, [ label: "Pleroma Admin Token", type: :group, description: - "Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the `admin_token` parameter (risky; use HTTP Basic Auth or OAuth-based authentication if possible)", + "Allows setting a token that can be used to authenticate requests with admin privileges without a normal user account token. Append the `admin_token` parameter to requests to utilize it. (Please reconsider using HTTP Basic Auth or OAuth-based authentication if possible)", children: [ %{ key: :admin_token, type: :string, description: "Admin token", suggestions: [ - "We recommend NOT setting the value do to increased security risk; if set, use a secure random long string or UUID (and change it as often as possible)" + "Please use a high entropy string or UUID" ] } ] -- cgit v1.2.3 From 58a4f350a8bc361d793cb96442f856362c18f195 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 6 May 2020 01:51:10 +0300 Subject: Refactor gun pooling and simplify adapter option insertion This patch refactors gun pooling to use Elixir process registry and simplifies adapter option insertion. Having the pool use process registry instead of a GenServer has a number of advantages: - Simpler code: the initial implementation adds about half the lines of code it deletes - Concurrency: unlike a GenServer, ETS-based registry can handle multiple checkout/checkin requests at the same time - Precise and easy idle connection clousure: current proposal for closing idle connections in the GenServer-based pool needs to filter through all connections once a minute and compare their last active time with closing time. With Elixir process registry this can be done by just using `Process.send_after`/`Process.cancel_timer` in the worker process. - Lower memory footprint: In my tests `gun-memory-leak` branch uses about 290mb on peak load (250 connections) and 235mb on idle (5-10 connections). Registry-based pool uses 210mb on idle and 240mb on peak load --- config/config.exs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 6fc84efc2..577ccc198 100644 --- a/config/config.exs +++ b/config/config.exs @@ -647,8 +647,10 @@ config :pleroma, Pleroma.Repo, prepare: :unnamed config :pleroma, :connections_pool, + reclaim_multiplier: 0.1, checkin_timeout: 250, max_connections: 250, + max_idle_time: 30_000, retry: 1, retry_timeout: 1000, await_up_timeout: 5_000 -- cgit v1.2.3 From 4128e3a84a2b6d75a8f92759e65ee673b47cec01 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 May 2020 22:16:02 +0300 Subject: HTTP: Implement max request limits --- config/config.exs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 577ccc198..dfc7a99d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -648,7 +648,8 @@ config :pleroma, Pleroma.Repo, config :pleroma, :connections_pool, reclaim_multiplier: 0.1, - checkin_timeout: 250, + connection_acquisition_wait: 250, + connection_acquisition_retries: 5, max_connections: 250, max_idle_time: 30_000, retry: 1, @@ -658,23 +659,19 @@ config :pleroma, :connections_pool, config :pleroma, :pools, federation: [ size: 50, - max_overflow: 10, - timeout: 150_000 + max_waiting: 10 ], media: [ size: 50, - max_overflow: 10, - timeout: 150_000 + max_waiting: 10 ], upload: [ size: 25, - max_overflow: 5, - timeout: 300_000 + max_waiting: 5 ], default: [ size: 10, - max_overflow: 2, - timeout: 10_000 + max_waiting: 2 ] config :pleroma, :hackney_pools, -- cgit v1.2.3 From 007843b75e0c7087dad1ef932224b21327d81793 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 23 Jun 2020 15:38:45 +0300 Subject: Add documentation for new connection pool settings and remove some `:retry_timeout` and `:retry` got removed because reconnecting on failure is something the new pool intentionally doesn't do. `:max_overflow` had to go in favor of `:max_waiting`, I didn't reuse the key because the settings are very different in their behaviour. `:checkin_timeout` got removed in favor of `:connection_acquisition_wait`, I didn't reuse the key because the settings are somewhat different. I didn't do any migrations/deprecation warnings/changelog entries because these settings were never in stable. --- config/description.exs | 156 +++++++++++++------------------------------------ 1 file changed, 39 insertions(+), 117 deletions(-) (limited to 'config') diff --git a/config/description.exs b/config/description.exs index afc4dcd79..f1c6773f1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3161,36 +3161,37 @@ config :pleroma, :config_description, [ description: "Advanced settings for `gun` connections pool", children: [ %{ - key: :checkin_timeout, + key: :connection_acquisition_wait, type: :integer, - description: "Timeout to checkin connection from pool. Default: 250ms.", - suggestions: [250] - }, - %{ - key: :max_connections, - type: :integer, - description: "Maximum number of connections in the pool. Default: 250 connections.", + description: + "Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. Default: 250ms.", suggestions: [250] }, %{ - key: :retry, + key: :connection_acquisition_retries, type: :integer, description: - "Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.", - suggestions: [1] + "Number of attempts to acquire the connection from the pool if it is overloaded. Default: 5", + suggestions: [5] }, %{ - key: :retry_timeout, + key: :max_connections, type: :integer, - description: - "Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.", - suggestions: [1000] + description: "Maximum number of connections in the pool. Default: 250 connections.", + suggestions: [250] }, %{ key: :await_up_timeout, type: :integer, description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.", suggestions: [5000] + }, + %{ + key: :reclaim_multiplier, + type: :integer, + description: + "Multiplier for the number of idle connection to be reclaimed if the pool is full. For example if the pool maxes out at 250 connections and this setting is set to 0.3, the pool will reclaim at most 75 idle connections if it's overloaded. Default: 0.1", + suggestions: [0.1] } ] }, @@ -3199,108 +3200,29 @@ config :pleroma, :config_description, [ key: :pools, type: :group, description: "Advanced settings for `gun` workers pools", - children: [ - %{ - key: :federation, - type: :keyword, - description: "Settings for federation pool.", - children: [ - %{ - key: :size, - type: :integer, - description: "Number workers in the pool.", - suggestions: [50] - }, - %{ - key: :max_overflow, - type: :integer, - description: "Number of additional workers if pool is under load.", - suggestions: [10] - }, - %{ - key: :timeout, - type: :integer, - description: "Timeout while `gun` will wait for response.", - suggestions: [150_000] - } - ] - }, - %{ - key: :media, - type: :keyword, - description: "Settings for media pool.", - children: [ - %{ - key: :size, - type: :integer, - description: "Number workers in the pool.", - suggestions: [50] - }, - %{ - key: :max_overflow, - type: :integer, - description: "Number of additional workers if pool is under load.", - suggestions: [10] - }, - %{ - key: :timeout, - type: :integer, - description: "Timeout while `gun` will wait for response.", - suggestions: [150_000] - } - ] - }, - %{ - key: :upload, - type: :keyword, - description: "Settings for upload pool.", - children: [ - %{ - key: :size, - type: :integer, - description: "Number workers in the pool.", - suggestions: [25] - }, - %{ - key: :max_overflow, - type: :integer, - description: "Number of additional workers if pool is under load.", - suggestions: [5] - }, - %{ - key: :timeout, - type: :integer, - description: "Timeout while `gun` will wait for response.", - suggestions: [300_000] - } - ] - }, - %{ - key: :default, - type: :keyword, - description: "Settings for default pool.", - children: [ - %{ - key: :size, - type: :integer, - description: "Number workers in the pool.", - suggestions: [10] - }, - %{ - key: :max_overflow, - type: :integer, - description: "Number of additional workers if pool is under load.", - suggestions: [2] - }, - %{ - key: :timeout, - type: :integer, - description: "Timeout while `gun` will wait for response.", - suggestions: [10_000] - } - ] - } - ] + children: + Enum.map([:federation, :media, :upload, :default], fn pool_name -> + %{ + key: pool_name, + type: :keyword, + description: "Settings for #{pool_name} pool.", + children: [ + %{ + key: :size, + type: :integer, + description: "Maximum number of concurrent requests in the pool.", + suggestions: [50] + }, + %{ + key: :max_waiting, + type: :integer, + description: + "Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made", + suggestions: [10] + } + ] + } + end) }, %{ group: :pleroma, -- cgit v1.2.3 From 9df59189747620c60173e6a67f8721971f123efd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 23 Jun 2020 15:52:57 +0300 Subject: config.exs: make gun the default again --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index dfc7a99d1..30b5e83bd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -172,7 +172,7 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, -- cgit v1.2.3 From 33747e9366ef7422c9b39ac360ad1d96405bc4fd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 6 Jul 2020 12:13:02 +0300 Subject: config.exs: set gun retries to 0 The new pooling code just removes the connection when it's down, there is no need to reconnect a connection that is just sitting idle, better just open a new one next time it's needed --- config/config.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 30b5e83bd..61406687a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -652,8 +652,7 @@ config :pleroma, :connections_pool, connection_acquisition_retries: 5, max_connections: 250, max_idle_time: 30_000, - retry: 1, - retry_timeout: 1000, + retry: 0, await_up_timeout: 5_000 config :pleroma, :pools, -- cgit v1.2.3 From c413649a8db26db742ff53c6c09a9a3b96e8cb6a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 Jul 2020 16:20:17 +0300 Subject: Bring back oban job pruning Closes #1945 --- config/config.exs | 1 + 1 file changed, 1 insertion(+) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 6fc84efc2..daeefdca3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -512,6 +512,7 @@ config :pleroma, Oban, attachments_cleanup: 5, new_users_digest: 1 ], + plugins: [Oban.Plugins.Pruner], crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, -- cgit v1.2.3