From 96bc9670082a9b86a4694fbaaba3bc300b436be2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 8 Apr 2020 16:01:03 +0300 Subject: [PATCH] closing idle gun connections --- config/config.exs | 2 ++ docs/configuration/cheatsheet.md | 9 ++++++-- lib/pleroma/pool/connections.ex | 48 +++++++++++++++++++++++++++++++++++----- test/pool/connections_test.exs | 34 ++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index ca9bbab64..f34a2d91c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -608,6 +608,8 @@ config :pleroma, Pleroma.Repo, config :pleroma, :connections_pool, checkin_timeout: 250, max_connections: 250, + max_idle_time: 10, + closing_idle_conns_interval: 10, retry: 1, retry_timeout: 1000, await_up_timeout: 5_000 diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 705c4c15e..e7a9b8836 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -394,15 +394,20 @@ For each pool, the options are: Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools. -For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000. -It will increase memory usage, but federation would work faster. +For big instances it's recommended to increase `max_connections` up to 500-1000. It will increase memory usage, but federation would work faster. * `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms. * `:max_connections` - maximum number of connections in the pool. Default: 250 connections. +* `:max_idle_time` - maximum of time, while connection can be idle. Default: 10 minutes. +* `:closing_idle_conns_interval` - interval between cleaning pool from idle connections. Default: 10 minutes. * `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1. * `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms. * `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms. +*If you are increasing `max_connections` setting, dont't forget to increase limit for file descriptors:* +* `installation/pleroma.service` - `LimitNOFILE` +* `installation/pleroma.supervisord` - `minfds` + ### :pools *For `gun` adapter* diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index acafe1bea..984057ab9 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Pool.Connections do @type domain :: String.t() @type conn :: Pleroma.Gun.Conn.t() + @type seconds :: pos_integer() @type t :: %__MODULE__{ conns: %{domain() => conn()}, @@ -26,7 +27,10 @@ defmodule Pleroma.Pool.Connections do end @impl true - def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + def init(opts) do + schedule_close_idle_conns() + {:ok, %__MODULE__{conns: %{}, opts: opts}} + end @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil def checkin(url, name) @@ -154,16 +158,16 @@ defmodule Pleroma.Pool.Connections do def handle_call(:unused_conns, _from, state) do unused_conns = state.conns - |> Enum.filter(&filter_conns/1) - |> Enum.sort(&sort_conns/2) + |> Enum.filter(&idle_conn?/1) + |> Enum.sort(&least_used/2) {:reply, unused_conns, state} end - defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true - defp filter_conns(_), do: false + defp idle_conn?({_, %{conn_state: :idle}}), do: true + defp idle_conn?(_), do: false - defp sort_conns({_, c1}, {_, c2}) do + defp least_used({_, c1}, {_, c2}) do c1.crf <= c2.crf and c1.last_reference <= c2.last_reference end @@ -265,6 +269,38 @@ defmodule Pleroma.Pool.Connections do {:noreply, state} end + @impl true + def handle_info({:close_idle_conns, max_idle_time}, state) do + closing_time = :os.system_time(:second) - max_idle_time + + idle_conns_keys = + state.conns + |> Enum.filter(&idle_more_than?(&1, closing_time)) + |> Enum.map(fn {key, %{conn: conn}} -> + Gun.close(conn) + key + end) + + schedule_close_idle_conns() + {:noreply, put_in(state.conns, Map.drop(state.conns, idle_conns_keys))} + end + + defp schedule_close_idle_conns do + max_idle_time = Config.get([:connections_pool, :max_idle_time], 10) * 60 + interval = Config.get([:connections_pool, :closing_idle_conns_interval], 10) * 60 * 1000 + Process.send_after(self(), {:close_idle_conns, max_idle_time}, interval) + end + + defp idle_more_than?( + {_, %{conn_state: :idle, last_reference: idle_since}}, + closing_time + ) + when closing_time >= idle_since do + true + end + + defp idle_more_than?(_, _), do: false + defp find_conn(conns, conn_pid) do Enum.find(conns, fn {_key, conn} -> conn.conn == conn_pid diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs index aeda54875..ad56e582f 100644 --- a/test/pool/connections_test.exs +++ b/test/pool/connections_test.exs @@ -757,4 +757,38 @@ defmodule Pleroma.Pool.ConnectionsTest do Connections.remove_conn(name, "1") assert Connections.count(name) == 0 end + + test "close_idle_conns/2", %{name: name} do + GunMock + |> expect(:close, fn _ -> :ok end) + |> allow(self(), name) + + Connections.add_conn(name, "1", %Conn{ + conn_state: :idle, + last_reference: now() - 30, + conn: self() + }) + + Connections.add_conn(name, "2", %Conn{ + conn_state: :idle, + last_reference: now() - 10, + conn: self() + }) + + Connections.add_conn(name, "3", %Conn{ + conn_state: :active, + conn: self() + }) + + name + |> Process.whereis() + |> send({:close_idle_conns, 15}) + + assert %Connections{ + conns: %{ + "3" => %Conn{}, + "2" => %Conn{} + } + } = Connections.get_state(name) + end end