@@ -735,28 +735,28 @@ config :pleroma, :connections_pool, | |||
max_connections: 250, | |||
max_idle_time: 30_000, | |||
retry: 0, | |||
await_up_timeout: 5_000 | |||
connect_timeout: 5_000 | |||
config :pleroma, :pools, | |||
federation: [ | |||
size: 50, | |||
max_waiting: 10, | |||
timeout: 10_000 | |||
recv_timeout: 10_000 | |||
], | |||
media: [ | |||
size: 50, | |||
max_waiting: 10, | |||
timeout: 10_000 | |||
recv_timeout: 10_000 | |||
], | |||
upload: [ | |||
size: 25, | |||
max_waiting: 5, | |||
timeout: 15_000 | |||
recv_timeout: 15_000 | |||
], | |||
default: [ | |||
size: 10, | |||
max_waiting: 2, | |||
timeout: 5_000 | |||
recv_timeout: 5_000 | |||
] | |||
config :pleroma, :hackney_pools, | |||
@@ -3377,7 +3377,7 @@ config :pleroma, :config_description, [ | |||
suggestions: [250] | |||
}, | |||
%{ | |||
key: :await_up_timeout, | |||
key: :connect_timeout, | |||
type: :integer, | |||
description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.", | |||
suggestions: [5000] | |||
@@ -3415,6 +3415,12 @@ config :pleroma, :config_description, [ | |||
description: | |||
"Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made", | |||
suggestions: [10] | |||
}, | |||
%{ | |||
key: :recv_timeout, | |||
type: :integer, | |||
description: "Timeout for the pool while gun will wait for response", | |||
suggestions: [10_000] | |||
} | |||
] | |||
} | |||
@@ -498,7 +498,7 @@ Settings for HTTP connection pool. | |||
* `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. | |||
* `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart. | |||
* `:max_connections` - Maximum number of connections in the pool. | |||
* `:await_up_timeout` - Timeout to connect to the host. | |||
* `:connect_timeout` - Timeout to connect to the host. | |||
* `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded. | |||
### :pools | |||
@@ -517,7 +517,7 @@ There are four pools used: | |||
For each pool, the options are: | |||
* `:size` - limit to how much requests can be concurrently executed. | |||
* `:timeout` - timeout while `gun` will wait for response | |||
* `:recv_timeout` - timeout while `gun` will wait for response | |||
* `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped. | |||
## Captcha | |||
@@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do | |||
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) | |||
with {:ok, %{status: 200, body: zip_body}} <- | |||
Pleroma.HTTP.get(url, [], | |||
adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000] | |||
) do | |||
Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do | |||
unzip(zip_body, dest) | |||
else | |||
e -> {:error, e} | |||
@@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do | |||
opts = | |||
opts | |||
|> Enum.into(%{}) | |||
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) | |||
|> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000) | |||
|> Map.put_new(:supervise, false) | |||
|> maybe_add_tls_opts(uri) | |||
@@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do | |||
with open_opts <- Map.delete(opts, :tls_opts), | |||
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), | |||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), | |||
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]), | |||
stream <- Gun.connect(conn, connect_opts), | |||
{:response, :fin, 200, _} <- Gun.await(conn, stream) do | |||
{:ok, conn} | |||
@@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do | |||
|> Map.put(:socks_opts, socks_opts) | |||
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), | |||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do | |||
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do | |||
{:ok, conn} | |||
else | |||
error -> | |||
@@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do | |||
host = Pleroma.HTTP.AdapterHelper.parse_host(host) | |||
with {:ok, conn} <- Gun.open(host, port, opts), | |||
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do | |||
{:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do | |||
{:ok, conn} | |||
else | |||
error -> | |||
@@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do | |||
@moduledoc """ | |||
Configure Tesla.Client with default and customized adapter options. | |||
""" | |||
@defaults [pool: :federation] | |||
@defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000] | |||
@type proxy_type() :: :socks4 | :socks5 | |||
@type host() :: charlist() | :inet.ip_address() | |||
@@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do | |||
require Logger | |||
@defaults [ | |||
connect_timeout: 5_000, | |||
domain_lookup_timeout: 5_000, | |||
tls_handshake_timeout: 5_000, | |||
retry: 1, | |||
retry_timeout: 1000, | |||
await_up_timeout: 5_000 | |||
retry_timeout: 1_000 | |||
] | |||
@type pool() :: :federation | :upload | :media | :default | |||
@@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do | |||
end | |||
defp put_timeout(opts) do | |||
{recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool])) | |||
# this is the timeout to receive a message from Gun | |||
Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) | |||
# `:timeout` key is used in Tesla | |||
Keyword.put(opts, :timeout, recv_timeout) | |||
end | |||
@spec pool_timeout(pool()) :: non_neg_integer() | |||
def pool_timeout(pool) do | |||
default = Config.get([:pools, :default, :timeout], 5_000) | |||
default = Config.get([:pools, :default, :recv_timeout], 5_000) | |||
Config.get([:pools, pool, :timeout], default) | |||
Config.get([:pools, pool, :recv_timeout], default) | |||
end | |||
@prefix Pleroma.Gun.ConnectionPool | |||
@@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do | |||
@behaviour Pleroma.HTTP.AdapterHelper | |||
@defaults [ | |||
connect_timeout: 10_000, | |||
recv_timeout: 20_000, | |||
follow_redirect: true, | |||
force_redirect: true, | |||
pool: :federation | |||
force_redirect: true | |||
] | |||
@spec options(keyword(), URI.t()) :: keyword() | |||
@@ -19,6 +16,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do | |||
|> Keyword.merge(config_opts) | |||
|> Keyword.merge(connection_opts) | |||
|> add_scheme_opts(uri) | |||
|> maybe_add_with_body() | |||
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) | |||
end | |||
@@ -27,4 +25,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do | |||
end | |||
defp add_scheme_opts(opts, _), do: opts | |||
defp maybe_add_with_body(opts) do | |||
if opts[:max_body] do | |||
Keyword.put(opts, :with_body, true) | |||
else | |||
opts | |||
end | |||
end | |||
end |
@@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do | |||
require Logger | |||
@options [ | |||
pool: :media | |||
pool: :media, | |||
recv_timeout: 10_000 | |||
] | |||
def perform(:prefetch, url) do | |||
Logger.debug("Prefetching #{inspect(url)}") | |||
opts = | |||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do | |||
Keyword.put(@options, :recv_timeout, 10_000) | |||
else | |||
@options | |||
end | |||
url | |||
|> MediaProxy.url() | |||
|> HTTP.get([], adapter: opts) | |||
|> HTTP.get([], adapter: @options) | |||
end | |||
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do | |||
@@ -5,7 +5,8 @@ | |||
defmodule Pleroma.Web.RelMe do | |||
@options [ | |||
pool: :media, | |||
max_body: 2_000_000 | |||
max_body: 2_000_000, | |||
recv_timeout: 2_000 | |||
] | |||
if Pleroma.Config.get(:env) == :test do | |||
@@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do | |||
def parse(_), do: {:error, "No URL provided"} | |||
defp parse_url(url) do | |||
opts = | |||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do | |||
Keyword.merge(@options, | |||
recv_timeout: 2_000, | |||
with_body: true | |||
) | |||
else | |||
@options | |||
end | |||
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- | |||
Pleroma.HTTP.get(url, [], adapter: opts), | |||
Pleroma.HTTP.get(url, [], adapter: @options), | |||
{:ok, html_tree} <- Floki.parse_document(html), | |||
data <- | |||
Floki.attribute(html_tree, "link[rel~=me]", "href") ++ | |||
@@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do | |||
alias Pleroma.Object | |||
alias Pleroma.Web.RichMedia.Parser | |||
@rich_media_options [ | |||
@options [ | |||
pool: :media, | |||
max_body: 2_000_000 | |||
max_body: 2_000_000, | |||
recv_timeout: 2_000 | |||
] | |||
@spec validate_page_url(URI.t() | binary()) :: :ok | :error | |||
defp validate_page_url(page_url) when is_binary(page_url) do | |||
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) | |||
validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) | |||
page_url | |||
|> Linkify.Parser.url?(validate_tld: validate_tld) | |||
@@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do | |||
def rich_media_get(url) do | |||
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] | |||
options = | |||
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do | |||
Keyword.merge(@rich_media_options, | |||
recv_timeout: 2_000, | |||
with_body: true | |||
) | |||
else | |||
@rich_media_options | |||
end | |||
Pleroma.HTTP.get(url, headers, adapter: options) | |||
Pleroma.HTTP.get(url, headers, adapter: @options) | |||
end | |||
end |