From 23d279e03ee1f7a1285614754738711359bc4b81 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 1 Aug 2019 17:28:00 +0300 Subject: [#1149] Replaced RetryQueue with oban-based retries. --- config/config.exs | 17 ++++++++++------- config/test.exs | 4 ++++ 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 17770640a..1bb325bf5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -440,13 +440,7 @@ config :pleroma, Pleroma.User, "web" ] -config :pleroma, Pleroma.Web.Federator.RetryQueue, - enabled: false, - max_jobs: 20, - initial_timeout: 30, - max_retries: 5 - -config :pleroma_job_queue, :queues, +job_queues = [ federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -454,6 +448,15 @@ config :pleroma_job_queue, :queues, transmogrifier: 20, scheduled_activities: 10, background: 5 +] + +config :pleroma_job_queue, :queues, job_queues + +config :pleroma, Oban, + repo: Pleroma.Repo, + verbose: false, + prune: {:maxage, 60 * 60 * 24 * 7}, + queues: job_queues config :pleroma, :fetch_initial_posts, enabled: false, diff --git a/config/test.exs b/config/test.exs index 92dca18bc..23d9bf779 100644 --- a/config/test.exs +++ b/config/test.exs @@ -62,6 +62,10 @@ config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock config :pleroma_job_queue, disabled: true +config :pleroma, Oban, + queues: false, + prune: :disabled + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, -- cgit v1.2.3 From b7fad8d395c2bd1afe445a370e539571f5ec0c18 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 9 Aug 2019 20:08:01 +0300 Subject: [#1149] Oban jobs implementation for :federator_incoming and :federator_outgoing queues. --- config/config.exs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 1bb325bf5..5fd64365c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -458,6 +458,13 @@ config :pleroma, Oban, prune: {:maxage, 60 * 60 * 24 * 7}, queues: job_queues +config :pleroma, :workers, + retries: [ + compile_time_default: 1, + federator_incoming: 5, + federator_outgoing: 5 + ] + config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 -- cgit v1.2.3 From c29686309eaf2cdae039ce813755c0e23cdc4a03 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 09:23:10 +0300 Subject: [#1149] Upgraded `oban` from 0.6.0 to 0.7.1. --- config/config.exs | 1 - 1 file changed, 1 deletion(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 9794997d9..1a6348bcd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -469,7 +469,6 @@ config :pleroma, Oban, config :pleroma, :workers, retries: [ - compile_time_default: 1, federator_incoming: 5, federator_outgoing: 5 ] -- cgit v1.2.3 From 581123f8bb703023cb652267a1fc34292f862852 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 18:28:23 +0300 Subject: [#1149] Introduced `quantum` job scheduler. Documentation & config changes. --- config/config.exs | 40 +++++++++++++++++++++++++++------------- config/test.exs | 2 -- 2 files changed, 27 insertions(+), 15 deletions(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 1a6348bcd..43d114d70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,6 +51,24 @@ config :pleroma, Pleroma.Repo, telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil +scheduled_jobs = + with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], + true <- digest_config[:active] do + [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + else + _ -> [] + end + +scheduled_jobs = + scheduled_jobs ++ + [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] + +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: scheduled_jobs + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -449,23 +467,19 @@ config :pleroma, Pleroma.User, "web" ] -job_queues = [ - federator_incoming: 50, - federator_outgoing: 50, - web_push: 50, - mailer: 10, - transmogrifier: 20, - scheduled_activities: 10, - background: 5 -] - -config :pleroma_job_queue, :queues, job_queues - config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, prune: {:maxage, 60 * 60 * 24 * 7}, - queues: job_queues + queues: [ + federator_incoming: 50, + federator_outgoing: 50, + web_push: 50, + mailer: 10, + transmogrifier: 20, + scheduled_activities: 10, + background: 5 + ] config :pleroma, :workers, retries: [ diff --git a/config/test.exs b/config/test.exs index a0fa67516..62f2a04d2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -61,8 +61,6 @@ config :web_push_encryption, :vapid_details, config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock -config :pleroma_job_queue, disabled: true - config :pleroma, Oban, queues: false, prune: :disabled -- cgit v1.2.3 From a90ea8ba1562818b025f677ffeea35f7ca08ddf2 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 31 Aug 2019 19:08:56 +0300 Subject: [#1149] Addressed code review comments (code style, jobs pruning etc.). --- config/config.exs | 2 +- config/test.exs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index da89aa3e9..6fb4a0969 100644 --- a/config/config.exs +++ b/config/config.exs @@ -470,7 +470,7 @@ config :pleroma, Pleroma.User, config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, - prune: {:maxage, 60 * 60 * 24 * 7}, + prune: {:maxlen, 1500}, queues: [ activity_expiration: 10, federator_incoming: 50, diff --git a/config/test.exs b/config/test.exs index 0ef809ac1..df512b5d7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -65,6 +65,8 @@ config :pleroma, Oban, queues: false, prune: :disabled +config :pleroma, Pleroma.Scheduler, jobs: [] + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, -- cgit v1.2.3 From b49085c156a6a4449c95c2c315f6250317122735 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 2 Sep 2019 14:57:40 +0300 Subject: [#1149] Refactoring: GenServer workers renamed to daemons, `use Oban.Worker` moved to helper. --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config') diff --git a/config/config.exs b/config/config.exs index 6fb4a0969..b742a650d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -54,7 +54,7 @@ config :pleroma, Pleroma.Repo, scheduled_jobs = with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] else _ -> [] end -- cgit v1.2.3 From e127b9ab6d01da48ebad188d2b9fcf7cb8a41578 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 14 Sep 2019 16:28:59 +0300 Subject: [#1149] Rewritten readme as config/description.exs. --- config/description.exs | 180 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 117 insertions(+), 63 deletions(-) (limited to 'config') diff --git a/config/description.exs b/config/description.exs index c5ae63915..be5eb0cc3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1778,87 +1778,141 @@ config :pleroma, :config_description, [ group: :pleroma_job_queue, key: :queues, type: :group, - description: "Pleroma Job Queue configuration: a list of queues with maximum concurrent jobs", + description: "[Deprecated] Replaced with `Oban`/`:queues` (keeping the same format)", + children: [] + }, + %{ + group: :pleroma, + key: Pleroma.Web.Federator.RetryQueue, + type: :group, + description: "[Deprecated] See `Oban` and `:workers` sections for configuration notes", children: [ %{ - key: :federator_outgoing, - type: :integer, - description: "Outgoing federation queue", - suggestions: [50] - }, - %{ - key: :federator_incoming, - type: :integer, - description: "Incoming federation queue", - suggestions: [50] - }, - %{ - key: :mailer, - type: :integer, - description: "Email sender queue, see Pleroma.Emails.Mailer", - suggestions: [10] - }, - %{ - key: :web_push, + key: :max_retries, type: :integer, - description: "Web push notifications queue", - suggestions: [50] - }, + description: "[Deprecated] Replaced as `Oban`/`:queues`/`:outgoing_federation` value", + suggestions: [] + } + ] + }, + %{ + group: :pleroma, + key: Oban, + type: :group, + description: """ + [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. + + Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), + it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, + otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings + (see https://github.com/sorentwo/oban/issues/52). + """, + children: [ %{ - key: :transmogrifier, - type: :integer, - description: "Transmogrifier queue", - suggestions: [20] + key: :repo, + type: :module, + description: "Application's Ecto repo", + suggestions: [Pleroma.Repo] }, %{ - key: :scheduled_activities, - type: :integer, - description: "Scheduled activities queue, see Pleroma.ScheduledActivities", - suggestions: [10] + key: :verbose, + type: :boolean, + description: "Logs verbose mode", + suggestions: [false, true] }, %{ - key: :activity_expiration, - type: :integer, - description: "Activity expiration queue", - suggestions: [10] + key: :prune, + type: [:atom, :tuple], + description: + "Non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning)", + suggestions: [:disabled, {:maxlen, 1500}, {:maxage, 60 * 60}] }, %{ - key: :background, - type: :integer, - description: "Background queue", - suggestions: [5] + key: :queues, + type: :keyword, + description: + "Background jobs queues (keys: queues, values: max numbers of concurrent jobs)", + suggestions: [ + [ + activity_expiration: 10, + background: 5, + federator_incoming: 50, + federator_outgoing: 50, + mailer: 10, + scheduled_activities: 10, + transmogrifier: 20, + web_push: 50 + ] + ], + children: [ + %{ + key: :activity_expiration, + type: :integer, + description: "Activity expiration queue", + suggestions: [10] + }, + %{ + key: :background, + type: :integer, + description: "Background queue", + suggestions: [5] + }, + %{ + key: :federator_incoming, + type: :integer, + description: "Incoming federation queue", + suggestions: [50] + }, + %{ + key: :federator_outgoing, + type: :integer, + description: "Outgoing federation queue", + suggestions: [50] + }, + %{ + key: :mailer, + type: :integer, + description: "Email sender queue, see Pleroma.Emails.Mailer", + suggestions: [10] + }, + %{ + key: :scheduled_activities, + type: :integer, + description: "Scheduled activities queue, see Pleroma.ScheduledActivities", + suggestions: [10] + }, + %{ + key: :transmogrifier, + type: :integer, + description: "Transmogrifier queue", + suggestions: [20] + }, + %{ + key: :web_push, + type: :integer, + description: "Web push notifications queue", + suggestions: [50] + } + ] } ] }, %{ group: :pleroma, - key: Pleroma.Web.Federator.RetryQueue, + key: :workers, type: :group, - description: "", + description: "Includes custom worker options not interpretable directly by `Oban`", children: [ %{ - key: :enabled, - type: :boolean, - description: "If set to true, failed federation jobs will be retried", - suggestions: [true, false] - }, - %{ - key: :max_jobs, - type: :integer, - description: "The maximum amount of parallel federation jobs running at the same time", - suggestions: [20] - }, - %{ - key: :initial_timeout, - type: :integer, - description: "The initial timeout in seconds", - suggestions: [30] - }, - %{ - key: :max_retries, - type: :integer, - description: "The maximum number of times a federation job is retried", - suggestions: [5] + key: :retries, + type: :keyword, + description: "Max retry attempts for failed jobs, per `Oban` queue", + suggestions: [ + [ + federator_incoming: 5, + federator_outgoing: 5 + ] + ] } ] }, -- cgit v1.2.3