@@ -39,6 +39,7 @@ defmodule Pleroma.Application do | |||||
# every time the application is restarted, so we disable module | # every time the application is restarted, so we disable module | ||||
# conflicts at runtime | # conflicts at runtime | ||||
Code.compiler_options(ignore_module_conflict: true) | Code.compiler_options(ignore_module_conflict: true) | ||||
Pleroma.Telemetry.Logger.attach() | |||||
Config.Holder.save_default() | Config.Holder.save_default() | ||||
Pleroma.HTML.compile_scrubbers() | Pleroma.HTML.compile_scrubbers() | ||||
Config.DeprecationWarnings.warn() | Config.DeprecationWarnings.warn() | ||||
@@ -18,8 +18,12 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do | |||||
case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do | case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do | ||||
{:error, :max_children} -> | {:error, :max_children} -> | ||||
case free_pool() do | case free_pool() do | ||||
:ok -> start_worker(opts) | |||||
:error -> {:error, :pool_full} | |||||
:ok -> | |||||
start_worker(opts) | |||||
:error -> | |||||
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) | |||||
{:error, :pool_full} | |||||
end | end | ||||
res -> | res -> | ||||
@@ -44,6 +48,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do | |||||
|> round | |> round | ||||
|> max(1) | |> max(1) | ||||
:telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ | |||||
max_connections: max_connections, | |||||
reclaim_max: reclaim_max | |||||
}) | |||||
# :ets.fun2ms( | |||||
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> | |||||
# {worker_pid, crf, last_reference} end) | |||||
unused_conns = | unused_conns = | ||||
Registry.select( | Registry.select( | ||||
@registry, | @registry, | ||||
@@ -55,17 +67,35 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do | |||||
case unused_conns do | case unused_conns do | ||||
[] -> | [] -> | ||||
:telemetry.execute( | |||||
[:pleroma, :connection_pool, :reclaim, :stop], | |||||
%{reclaimed_count: 0}, | |||||
%{ | |||||
max_connections: max_connections | |||||
} | |||||
) | |||||
exit(:no_unused_conns) | exit(:no_unused_conns) | ||||
unused_conns -> | unused_conns -> | ||||
unused_conns | |||||
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> | |||||
crf1 <= crf2 and last_reference1 <= last_reference2 | |||||
end) | |||||
|> Enum.take(reclaim_max) | |||||
reclaimed = | |||||
unused_conns | |||||
|> Enum.sort(fn {_pid1, crf1, last_reference1}, | |||||
{_pid2, crf2, last_reference2} -> | |||||
crf1 <= crf2 and last_reference1 <= last_reference2 | |||||
end) | |||||
|> Enum.take(reclaim_max) | |||||
reclaimed | |||||
|> Enum.each(fn {pid, _, _} -> | |> Enum.each(fn {pid, _, _} -> | ||||
DynamicSupervisor.terminate_child(__MODULE__, pid) | DynamicSupervisor.terminate_child(__MODULE__, pid) | ||||
end) | end) | ||||
:telemetry.execute( | |||||
[:pleroma, :connection_pool, :reclaim, :stop], | |||||
%{reclaimed_count: Enum.count(reclaimed)}, | |||||
%{max_connections: max_connections} | |||||
) | |||||
end | end | ||||
end) | end) | ||||
@@ -0,0 +1,62 @@ | |||||
defmodule Pleroma.Telemetry.Logger do | |||||
@moduledoc "Transforms Pleroma telemetry events to logs" | |||||
require Logger | |||||
@events [ | |||||
[:pleroma, :connection_pool, :reclaim, :start], | |||||
[:pleroma, :connection_pool, :reclaim, :stop], | |||||
[:pleroma, :connection_pool, :provision_failure] | |||||
] | |||||
def attach do | |||||
:telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) | |||||
end | |||||
# Passing anonymous functions instead of strings to logger is intentional, | |||||
# that way strings won't be concatenated if the message is going to be thrown | |||||
# out anyway due to higher log level configured | |||||
def handle_event( | |||||
[:pleroma, :connection_pool, :reclaim, :start], | |||||
_, | |||||
%{max_connections: max_connections, reclaim_max: reclaim_max}, | |||||
_ | |||||
) do | |||||
Logger.debug(fn -> | |||||
"Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{ | |||||
reclaim_max | |||||
} connections" | |||||
end) | |||||
end | |||||
def handle_event( | |||||
[:pleroma, :connection_pool, :reclaim, :stop], | |||||
%{reclaimed_count: 0}, | |||||
_, | |||||
_ | |||||
) do | |||||
Logger.error(fn -> | |||||
"Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts" | |||||
end) | |||||
end | |||||
def handle_event( | |||||
[:pleroma, :connection_pool, :reclaim, :stop], | |||||
%{reclaimed_count: reclaimed_count}, | |||||
_, | |||||
_ | |||||
) do | |||||
Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end) | |||||
end | |||||
def handle_event( | |||||
[:pleroma, :connection_pool, :provision_failure], | |||||
%{opts: [key | _]}, | |||||
_, | |||||
_ | |||||
) do | |||||
Logger.error(fn -> | |||||
"Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" | |||||
end) | |||||
end | |||||
end |