Configurable limits for ConcurrentLimiter for Pleroma.Web.RichMedia.Helpers & Pleroma.Web.MediaProxyWarmingPolicy See merge request pleroma/pleroma!3248groups
@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). | |||||
- OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved. | - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved. | ||||
- OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc. | - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc. | ||||
- Ability to set ActivityPub aliases for follower migration. | - Ability to set ActivityPub aliases for follower migration. | ||||
- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy | |||||
<details> | <details> | ||||
<summary>API Changes</summary> | <summary>API Changes</summary> | ||||
@@ -832,6 +832,11 @@ config :pleroma, Pleroma.User.Backup, | |||||
limit_days: 7, | limit_days: 7, | ||||
dir: nil | dir: nil | ||||
config :pleroma, ConcurrentLimiter, [ | |||||
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, | |||||
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]} | |||||
] | |||||
# Import environment specific config. This must remain at the bottom | # Import environment specific config. This must remain at the bottom | ||||
# of this file so it overrides the configuration defined above. | # of this file so it overrides the configuration defined above. | ||||
import_config "#{Mix.env()}.exs" | import_config "#{Mix.env()}.exs" |
@@ -3330,5 +3330,53 @@ config :pleroma, :config_description, [ | |||||
suggestions: [:text, :protobuf] | suggestions: [:text, :protobuf] | ||||
} | } | ||||
] | ] | ||||
}, | |||||
%{ | |||||
group: :pleroma, | |||||
key: ConcurrentLimiter, | |||||
type: :group, | |||||
description: "Limits configuration for background tasks.", | |||||
children: [ | |||||
%{ | |||||
key: Pleroma.Web.RichMedia.Helpers, | |||||
type: :keyword, | |||||
description: "Concurrent limits configuration for getting RichMedia for activities.", | |||||
suggestions: [max_running: 5, max_waiting: 5], | |||||
children: [ | |||||
%{ | |||||
key: :max_running, | |||||
type: :integer, | |||||
description: "Max running concurrently jobs.", | |||||
suggestion: [5] | |||||
}, | |||||
%{ | |||||
key: :max_waiting, | |||||
type: :integer, | |||||
description: "Max waiting jobs.", | |||||
suggestion: [5] | |||||
} | |||||
] | |||||
}, | |||||
%{ | |||||
key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, | |||||
type: :keyword, | |||||
description: "Concurrent limits configuration for MediaProxyWarmingPolicy.", | |||||
suggestions: [max_running: 5, max_waiting: 5], | |||||
children: [ | |||||
%{ | |||||
key: :max_running, | |||||
type: :integer, | |||||
description: "Max running concurrently jobs.", | |||||
suggestion: [5] | |||||
}, | |||||
%{ | |||||
key: :max_waiting, | |||||
type: :integer, | |||||
description: "Max waiting jobs.", | |||||
suggestion: [5] | |||||
} | |||||
] | |||||
} | |||||
] | |||||
} | } | ||||
] | ] |
@@ -1110,3 +1110,15 @@ Settings to enable and configure expiration for ephemeral activities | |||||
* `:enabled` - enables ephemeral activities creation | * `:enabled` - enables ephemeral activities creation | ||||
* `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. | * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. | ||||
## ConcurrentLimiter | |||||
Settings to restrict concurrently running jobs. Jobs which can be configured: | |||||
* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities | |||||
* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy | |||||
Each job has these settings: | |||||
* `:max_running` - max concurrently runnings jobs | |||||
* `:max_waiting` - max waiting jobs |
@@ -297,7 +297,16 @@ defmodule Pleroma.Application do | |||||
@spec limiters_setup() :: :ok | @spec limiters_setup() :: :ok | ||||
def limiters_setup do | def limiters_setup do | ||||
[Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] | |||||
|> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) | |||||
config = Config.get(ConcurrentLimiter, []) | |||||
[Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] | |||||
|> Enum.each(fn module -> | |||||
mod_config = Keyword.get(config, module, []) | |||||
max_running = Keyword.get(mod_config, :max_running, 5) | |||||
max_waiting = Keyword.get(mod_config, :max_waiting, 5) | |||||
ConcurrentLimiter.new(module, max_running, max_waiting) | |||||
end) | |||||
end | end | ||||
end | end |
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do | |||||
if Pleroma.Config.get(:env) == :test do | if Pleroma.Config.get(:env) == :test do | ||||
fetch(prefetch_url) | fetch(prefetch_url) | ||||
else | else | ||||
ConcurrentLimiter.limit(MediaProxy, fn -> | |||||
ConcurrentLimiter.limit(__MODULE__, fn -> | |||||
Task.start(fn -> fetch(prefetch_url) end) | Task.start(fn -> fetch(prefetch_url) end) | ||||
end) | end) | ||||
end | end | ||||