fix after rebase

- Pleroma start is splitted into three `phases`
- added DynamicSupervisor, which starts startup dependencies

lib/mix/pleroma.ex

@@ -4,7 +4,6 @@
 
 defmodule Mix.Pleroma do
   @apps [
-    :restarter,
     :ecto,
     :ecto_sql,
     :postgrex,
@@ -16,11 +15,14 @@ defmodule Mix.Pleroma do
     :fast_html,
     :oban
   ]
+
   @cachex_children ["object", "user", "scrubber", "web_resp"]
+
   @doc "Common functions to be reused in mix tasks"
+  @spec start_pleroma() :: {:ok, pid()}
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
-    Pleroma.Config.Oban.warn()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
     Pleroma.Application.limiters_setup()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
@@ -47,37 +49,27 @@ defmodule Mix.Pleroma do
       plugins: []
     ]
 
-    children =
-      [
-        Pleroma.Repo,
-        Pleroma.Emoji,
-        {Pleroma.Config.TransferTask, false},
-        Pleroma.Web.Endpoint,
-        {Oban, oban_config},
-        {Majic.Pool,
-         [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
-      ] ++
-        http_children(adapter)
+    children = [
+      Pleroma.Repo,
+      Supervisor.child_spec({Task, &Pleroma.Application.Environment.load_from_db_and_update/0},
+        id: :update_env
+      ),
+      Pleroma.Emoji,
+      Pleroma.Web.Endpoint,
+      {Oban, oban_config},
+      {Majic.Pool,
+       [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
+    ]
 
-    cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
+    children = [Pleroma.Application.StartUpDependencies.adapter_module() | children]
+
+    cachex_children =
+      Enum.map(@cachex_children, &Pleroma.Application.StartUpDependencies.cachex_spec({&1, []}))
 
     Supervisor.start_link(children ++ cachex_children,
       strategy: :one_for_one,
       name: Pleroma.Supervisor
     )
-
-    if Pleroma.Config.get(:env) not in [:test, :benchmark] do
-      pleroma_rebooted?()
-    end
-  end
-
-  defp pleroma_rebooted? do
-    if Restarter.Pleroma.rebooted?() do
-      :ok
-    else
-      Process.sleep(10)
-      pleroma_rebooted?()
-    end
   end
 
   def load_pleroma do
@@ -129,11 +121,4 @@ defmodule Mix.Pleroma do
   def escape_sh_path(path) do
     ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
   end
-
-  defp http_children(Tesla.Adapter.Gun) do
-    Pleroma.Gun.ConnectionPool.children() ++
-      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
-  end
-
-  defp http_children(_), do: []
 end

lib/mix/tasks/pleroma/config.ex

@@ -237,17 +237,15 @@ defmodule Mix.Tasks.Pleroma.Config do
         |> elem(0)
 
       custom_config
-      |> Keyword.keys()
-      |> Enum.each(&create(&1, custom_config))
+      |> Pleroma.Config.Loader.filter()
+      |> Enum.each(&create/1)
     else
       shell_info("To migrate settings, you must define custom settings in #{config_file}.")
     end
   end
 
-  defp create(group, settings) do
-    group
-    |> Pleroma.Config.Loader.filter_group(settings)
-    |> Enum.each(fn {key, value} ->
+  defp create({group, settings}) do
+    Enum.each(settings, fn {key, value} ->
       {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
 
       shell_info("Settings for key #{key} migrated.")

lib/mix/tasks/pleroma/docs.ex

@@ -32,7 +32,7 @@ defmodule Mix.Tasks.Pleroma.Docs do
   defp do_run(implementation) do
     start_pleroma()
 
-    with descriptions <- Pleroma.Config.Loader.read("config/description.exs"),
+    with descriptions <- Pleroma.Config.Loader.read!("config/description.exs"),
          {:ok, file_path} <-
            Pleroma.Docs.Generator.process(
              implementation,

lib/pleroma/application.ex

@@ -5,22 +5,25 @@
 defmodule Pleroma.Application do
   use Application
 
-  import Cachex.Spec
+  require Logger
 
   alias Pleroma.Config
 
-  require Logger
-
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
   @repository Mix.Project.config()[:source_url]
   @mix_env Mix.env()
+  @dynamic_supervisor Pleroma.Application.Supervisor
+
+  @type env() :: :test | :benchmark | :dev | :prod
 
   def name, do: @name
   def version, do: @version
   def named_version, do: @name <> " " <> @version
   def repository, do: @repository
+  def dynamic_supervisor, do: @dynamic_supervisor
 
+  @spec user_agent() :: String.t()
   def user_agent do
     if Process.whereis(Pleroma.Web.Endpoint) do
       case Config.get([:http, :user_agent], :default) do
@@ -37,82 +40,60 @@ defmodule Pleroma.Application do
     end
   end
 
-  # See http://elixir-lang.org/docs/stable/elixir/Application.html
-  # for more information on OTP Applications
+  @doc """
+  Under main supervisor is started DynamicSupervisor, which later starts pleroma startup dependencies.
+  Pleroma start is splitted into three `phases`:
+    - running prestart requirements (runtime compilation, warnings, deprecations, monitoring, etc.)
+    - loading and updating environment (if database config is used and enabled)
+    - starting dependencies
+  """
+  @impl true
   def start(_type, _args) do
+    children = [
+      {DynamicSupervisor, strategy: :one_for_one, name: @dynamic_supervisor},
+      {Pleroma.Application.ConfigDependentDeps, [dynamic_supervisor: @dynamic_supervisor]},
+      Pleroma.Repo
+    ]
+
+    {:ok, main_supervisor} =
+      Supervisor.start_link(children, strategy: :one_for_one, name: Pleroma.Supervisor)
+
+    run_prestart_requirements()
+
+    Pleroma.Application.Environment.load_from_db_and_update(pleroma_start: true)
+
+    Pleroma.Application.StartUpDependencies.start_all(@mix_env)
+
+    {:ok, main_supervisor}
+  end
+
+  defp run_prestart_requirements do
     # Scrubbers are compiled at runtime and therefore will cause a conflict
     # every time the application is restarted, so we disable module
     # conflicts at runtime
     Code.compiler_options(ignore_module_conflict: true)
+
     # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
     # due to protocol consolidation warnings
     Code.compiler_options(warnings_as_errors: false)
-    Pleroma.Telemetry.Logger.attach()
-    Config.Holder.save_default()
+
+    # compilation in runtime
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.Oban.warn()
+    compile_custom_modules()
+    Pleroma.Docs.JSON.compile()
+
+    # telemetry and prometheus
+    Pleroma.Telemetry.Logger.attach()
+    setup_instrumenters()
+
+    Config.Holder.save_default()
     Config.DeprecationWarnings.warn()
     Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
-    Pleroma.ApplicationRequirements.verify!()
-    setup_instrumenters()
-    load_custom_modules()
-    Pleroma.Docs.JSON.compile()
+
     limiters_setup()
-
-    adapter = Application.get_env(:tesla, :adapter)
-
-    if adapter == Tesla.Adapter.Gun do
-      if version = Pleroma.OTPVersion.version() do
-        [major, minor] =
-          version
-          |> String.split(".")
-          |> Enum.map(&String.to_integer/1)
-          |> Enum.take(2)
-
-        if (major == 22 and minor < 2) or major < 22 do
-          raise "
-            !!!OTP VERSION WARNING!!!
-            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
-            "
-        end
-      else
-        raise "
-          !!!OTP VERSION WARNING!!!
-          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
-          "
-      end
-    end
-
-    # Define workers and child supervisors to be supervised
-    children =
-      [
-        Pleroma.Repo,
-        Config.TransferTask,
-        Pleroma.Emoji,
-        Pleroma.Web.Plugs.RateLimiter.Supervisor
-      ] ++
-        cachex_children() ++
-        http_children(adapter, @mix_env) ++
-        [
-          Pleroma.Stats,
-          Pleroma.JobQueueMonitor,
-          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
-          {Oban, Config.get(Oban)},
-          Pleroma.Web.Endpoint
-        ] ++
-        task_children(@mix_env) ++
-        dont_run_in_test(@mix_env) ++
-        chat_child(chat_enabled?()) ++
-        [Pleroma.Gopher.Server]
-
-    # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
-    # for other strategies and supported options
-    opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    result = Supervisor.start_link(children, opts)
-
     set_postgres_server_version()
 
-    result
+    Pleroma.Application.Requirements.verify!()
   end
 
   defp set_postgres_server_version do
@@ -132,7 +113,7 @@ defmodule Pleroma.Application do
     :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
   end
 
-  def load_custom_modules do
+  defp compile_custom_modules do
     dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
@@ -177,128 +158,6 @@ defmodule Pleroma.Application do
     PrometheusPhx.setup()
   end
 
-  defp cachex_children do
-    [
-      build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
-      build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
-      build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
-      build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
-      build_cachex("scrubber", limit: 2500),
-      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
-      build_cachex("web_resp", limit: 2500),
-      build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
-      build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
-      build_cachex("chat_message_id_idempotency_key",
-        expiration: chat_message_id_idempotency_key_expiration(),
-        limit: 500_000
-      )
-    ]
-  end
-
-  defp emoji_packs_expiration,
-    do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
-
-  defp idempotency_expiration,
-    do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
-
-  defp chat_message_id_idempotency_key_expiration,
-    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
-
-  defp seconds_valid_interval,
-    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
-
-  @spec build_cachex(String.t(), keyword()) :: map()
-  def build_cachex(type, opts),
-    do: %{
-      id: String.to_atom("cachex_" <> type),
-      start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
-      type: :worker
-    }
-
-  defp chat_enabled?, do: Config.get([:chat, :enabled])
-
-  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
-
-  defp dont_run_in_test(_) do
-    [
-      {Registry,
-       [
-         name: Pleroma.Web.Streamer.registry(),
-         keys: :duplicate,
-         partitions: System.schedulers_online()
-       ]}
-    ] ++ background_migrators()
-  end
-
-  defp background_migrators do
-    [
-      Pleroma.Migrators.HashtagsTableMigrator
-    ]
-  end
-
-  defp chat_child(true) do
-    [
-      Pleroma.Web.ChatChannel.ChatChannelState,
-      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
-    ]
-  end
-
-  defp chat_child(_), do: []
-
-  defp task_children(:test) do
-    [
-      %{
-        id: :web_push_init,
-        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-        restart: :temporary
-      }
-    ]
-  end
-
-  defp task_children(_) do
-    [
-      %{
-        id: :web_push_init,
-        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-        restart: :temporary
-      },
-      %{
-        id: :internal_fetch_init,
-        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
-        restart: :temporary
-      }
-    ]
-  end
-
-  # start hackney and gun pools in tests
-  defp http_children(_, :test) do
-    http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil)
-  end
-
-  defp http_children(Tesla.Adapter.Hackney, _) do
-    pools = [:federation, :media]
-
-    pools =
-      if Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload | pools]
-      else
-        pools
-      end
-
-    for pool <- pools do
-      options = Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
-  defp http_children(Tesla.Adapter.Gun, _) do
-    Pleroma.Gun.ConnectionPool.children() ++
-      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
-  end
-
-  defp http_children(_, _), do: []
-
   @spec limiters_setup() :: :ok
   def limiters_setup do
     config = Config.get(ConcurrentLimiter, [])

lib/pleroma/application/chat_supervisor.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.ChatSupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_args)
+  end
+
+  def init(_) do
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
+    |> Supervisor.init(strategy: :one_for_one)
+  end
+end

lib/pleroma/application/config_dependent_deps.ex

@@ -0,0 +1,244 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.ConfigDependentDeps do
+  use GenServer
+
+  require Logger
+
+  @config_path_mods_relation [
+    {{:pleroma, :chat}, Pleroma.Application.ChatSupervisor},
+    {{:pleroma, Oban}, Oban},
+    {{:pleroma, :rate_limit}, Pleroma.Web.Plugs.RateLimiter.Supervisor},
+    {{:pleroma, :streamer}, Pleroma.Web.Streamer.registry()},
+    {{:pleroma, :pools}, Pleroma.Gun.GunSupervisor},
+    {{:pleroma, :connections_pool}, Pleroma.Gun.GunSupervisor},
+    {{:pleroma, :hackney_pools}, Pleroma.HTTP.HackneySupervisor},
+    {{:pleroma, :gopher}, Pleroma.Gopher.Server},
+    {{:pleroma, Pleroma.Captcha, [:seconds_valid]}, Pleroma.Web.Endpoint},
+    {{:pleroma, Pleroma.Upload, [:proxy_remote]},
+     Pleroma.Application.StartUpDependencies.adapter_module()},
+    {{:pleroma, :instance, [:upload_limit]}, Pleroma.Web.Endpoint},
+    {{:pleroma, :fed_sockets, [:enabled]}, Pleroma.Web.Endpoint},
+    {:eshhd, :eshhd},
+    {:ex_aws, :ex_aws}
+  ]
+
+  def start_link(opts) do
+    opts = Keyword.put_new(opts, :relations, @config_path_mods_relation)
+
+    GenServer.start_link(__MODULE__, opts, name: opts[:name] || __MODULE__)
+  end
+
+  @impl true
+  def init(opts) do
+    init_state = %{
+      dynamic_supervisor: opts[:dynamic_supervisor],
+      relations: opts[:relations],
+      reboot_paths: [],
+      pids: %{}
+    }
+
+    {:ok, init_state}
+  end
+
+  def start_dependency(module, server \\ __MODULE__) do
+    GenServer.call(server, {:start_dependency, module})
+  end
+
+  def need_reboot?(server \\ __MODULE__) do
+    GenServer.call(server, :need_reboot?)
+  end
+
+  def restart_dependencies(server \\ __MODULE__) do
+    GenServer.call(server, :restart_dependencies)
+  end
+
+  def clear_state(server \\ __MODULE__) do
+    GenServer.call(server, :clear_state)
+  end
+
+  def save_config_paths_for_restart(changes, server \\ __MODULE__) do
+    GenServer.call(server, {:save_config_paths, changes})
+  end
+
+  @impl true
+  def handle_call({:start_dependency, module}, _, state) do
+    {result, state} =
+      with {pid, state} when is_pid(pid) <- start_module(module, state) do
+        {{:ok, pid}, state}
+      else
+        error -> {error, state}
+      end
+
+    {:reply, result, state}
+  end
+
+  @impl true
+  def handle_call(:need_reboot?, _, state) do
+    {:reply, state[:reboot_paths] != [], state}
+  end
+
+  @impl true
+  def handle_call(:restart_dependencies, _, state) do
+    {paths, state} = Map.get_and_update(state, :reboot_paths, &{&1, []})
+    started_apps = Application.started_applications()
+
+    {result, state} =
+      Enum.reduce_while(paths, {:ok, state}, fn
+        path, {:ok, acc} when is_tuple(path) ->
+          case restart(path, acc, acc[:pids][path], with_terminate: true) do
+            {pid, state} when is_pid(pid) ->
+              {:cont, {:ok, state}}
+
+            :ignore ->
+              Logger.info("path #{inspect(path)} is ignored.")
+              {:cont, {:ok, acc}}
+
+            error ->
+              {:halt, {error, acc}}
+          end
+
+        app, {:ok, acc}
+        when is_atom(app) and app not in [:logger, :quack, :pleroma, :prometheus, :postgrex] ->
+          restart_app(app, started_apps)
+          {:cont, {:ok, acc}}
+      end)
+
+    {:reply, result, state}
+  end
+
+  @impl true
+  def handle_call(:clear_state, _, state) do
+    state =
+      state
+      |> Map.put(:reboot_paths, [])
+      |> Map.put(:pids, %{})
+
+    {:reply, :ok, state}
+  end
+
+  @impl true
+  def handle_call({:save_config_paths, changes}, _, state) do
+    paths =
+      Enum.reduce(changes, state[:reboot_paths], fn
+        %{group: group, key: key, value: value}, acc ->
+          with {path, _} <- find_relation(state[:relations], group, key, value) do
+            if path not in acc do
+              [path | acc]
+            else
+              acc
+            end
+          else
+            _ ->
+              acc
+          end
+      end)
+
+    {:reply, paths, put_in(state[:reboot_paths], paths)}
+  end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do
+    updated_state =
+      with {path, ^pid} <-
+             Enum.find(state[:pids], fn {_, registered_pid} -> registered_pid == pid end) do
+        {_new_pid, new_state} = restart(path, state, pid)
+        new_state
+      else
+        _ -> state
+      end
+
+    {:noreply, updated_state}
+  end
+
+  defp start_module(module, state) do
+    with {:ok, relations} <- find_relations(state[:relations], module) do
+      start_module(module, relations, state)
+    end
+  end
+
+  defp start_module(module, relations, state) do
+    spec =
+      module
+      |> Pleroma.Application.StartUpDependencies.spec()
+      |> Supervisor.child_spec(restart: :temporary)
+
+    with {:ok, pid} <-
+           DynamicSupervisor.start_child(
+             state[:dynamic_supervisor],
+             spec
+           ) do
+      pids = Map.new(relations, fn {path, _} -> {path, pid} end)
+      Process.monitor(pid)
+      {pid, put_in(state[:pids], Map.merge(state[:pids], pids))}
+    end
+  end
+
+  defp restart(path, state, pid, opts \\ [])
+
+  defp restart(path, state, nil, _) do
+    with {_, module} <- find_relation(state[:relations], path) do
+      start_module(module, state)
+    end
+  end
+
+  defp restart(path, state, pid, opts) when is_pid(pid) do
+    with {_, module} <- find_relation(state[:relations], path),
+         {:ok, relations} <- find_relations(state[:relations], module) do
+      if opts[:with_terminate] do
+        :ok = DynamicSupervisor.terminate_child(state[:dynamic_supervisor], pid)
+      end
+
+      paths_for_remove = Enum.map(relations, fn {path, _} -> path end)
+      state = put_in(state[:pids], Map.drop(state[:pids], paths_for_remove))
+
+      start_module(module, relations, state)
+    end
+  end
+
+  defp restart_app(app, started_applications) do
+    with {^app, _, _} <- List.keyfind(started_applications, app, 0) do
+      :ok = Application.stop(app)
+      :ok = Application.start(app)
+    else
+      nil ->
+        Logger.info("#{app} is not started.")
+
+      error ->
+        error
+        |> inspect()
+        |> Logger.error()
+    end
+  end
+
+  defp find_relations(relations, module) do
+    case Enum.filter(relations, fn {_, mod} -> mod == module end) do
+      [] ->
+        {:error, :relations_not_found}
+
+      relations ->
+        {:ok, relations}
+    end
+  end
+
+  defp find_relation(relations, group, key, value) do
+    Enum.find(relations, fn
+      {g, _} when is_atom(g) ->
+        g == group
+
+      {{g, k}, _} ->
+        g == group and k == key
+
+      {{g, k, subkeys}, _} ->
+        g == group and k == key and Enum.any?(Keyword.keys(value), &(&1 in subkeys))
+    end)
+  end
+
+  def find_relation(relations, path) do
+    with nil <- Enum.find(relations, fn {key, _} -> key == path end) do
+      {:error, :relation_not_found}
+    end
+  end
+end

lib/pleroma/application/environment.ex

@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.Environment do
+  @moduledoc """
+  Overwrites environment config with settings from config file or database.
+  """
+
+  require Logger
+
+  @spec load_from_db_and_update(keyword()) :: :ok
+  def load_from_db_and_update(opts \\ []) do
+    Pleroma.ConfigDB.all()
+    |> update(opts)
+  end
+
+  @spec update([Pleroma.ConfigDB.t()], keyword()) :: :ok
+  def update(changes, opts \\ []) when is_list(changes) do
+    if Pleroma.Config.get(:configurable_from_database) do
+      defaults = Pleroma.Config.Holder.default_config()
+
+      configure_logger_and_quack(changes, defaults)
+
+      changes
+      |> Enum.map(fn config ->
+        {_, merged_value} = Pleroma.ConfigDB.merge_with_default(config, defaults)
+
+        %{config | value: merged_value}
+      end)
+      |> Enum.each(&update_env(&1))
+
+      cond do
+        # restart only apps on pleroma start
+        opts[:pleroma_start] ->
+          changes
+          |> Enum.filter(fn %{group: group} ->
+            group not in [:logger, :quack, :pleroma, :prometheus, :postgrex]
+          end)
+          |> Pleroma.Application.ConfigDependentDeps.save_config_paths_for_restart()
+
+          Pleroma.Application.ConfigDependentDeps.restart_dependencies()
+
+        opts[:only_update] ->
+          Pleroma.Application.ConfigDependentDeps.save_config_paths_for_restart(changes)
+
+        true ->
+          nil
+      end
+    end
+
+    :ok
+  end
+
+  defp configure_logger_and_quack(changes, defaults) do
+    {logger_changes, quack_changes} =
+      changes
+      |> Enum.filter(fn %{group: group} -> group in [:logger, :quack] end)
+      |> Enum.split_with(fn %{group: group} -> group == :logger end)
+
+    quack_config = to_keyword(quack_changes)
+
+    if quack_config != [] do
+      merged = Keyword.merge(defaults[:quack], quack_config)
+      Logger.configure_backend(Quack.Logger, merged)
+    end
+
+    logger_config = to_keyword(logger_changes)
+
+    if logger_config != [] do
+      merged = Keyword.merge(defaults, logger_config)
+
+      if logger_config[:backends] do
+        Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
+
+        Enum.each(merged[:backends], &Logger.add_backend/1)
+      end
+
+      if logger_config[:console] do
+        console = merged[:console]
+        console = put_in(console[:format], console[:format] <> "\n")
+
+        Logger.configure_backend(:console, console)
+      end
+
+      if logger_config[:ex_syslogger] do
+        Logger.configure_backend({ExSyslogger, :ex_syslogger}, merged[:ex_syslogger])
+      end
+
+      Logger.configure(merged)
+    end
+  end
+
+  defp to_keyword(changes) do
+    Enum.reduce(changes, [], fn
+      %{key: key, value: value}, acc ->
+        Keyword.put(acc, key, value)
+    end)
+  end
+
+  defp update_env(%{group: group, key: key, value: nil}), do: Application.delete_env(group, key)
+
+  defp update_env(%{group: group, key: key, value: config}) do
+    Application.put_env(group, key, config)
+  end
+end

lib/pleroma/application/requirements.ex

@@ -2,9 +2,9 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ApplicationRequirements do
+defmodule Pleroma.Application.Requirements do
   @moduledoc """
-  The module represents the collection of validations to runs before start server.
+  Module contains collection of requirements before server start.
   """
 
   defmodule VerifyError, do: defexception([:message])
@@ -18,6 +18,8 @@ defmodule Pleroma.ApplicationRequirements do
 
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
+    adapter = Application.get_env(:tesla, :adapter)
+
     :ok
     |> check_system_commands!()
     |> check_confirmation_accounts!()
@@ -25,11 +27,12 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_welcome_message_config!()
     |> check_rum!()
     |> check_repo_pool_size!()
-    |> handle_result()
+    |> check_otp_version!(adapter)
+    |> handle_result!()
   end
 
-  defp handle_result(:ok), do: :ok
-  defp handle_result({:error, message}), do: raise(VerifyError, message: message)
+  defp handle_result!(:ok), do: :ok
+  defp handle_result!({:error, message}), do: raise(VerifyError, message: message)
 
   defp check_welcome_message_config!(:ok) do
     if Pleroma.Config.get([:welcome, :email, :enabled], false) and
@@ -160,9 +163,9 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"),
-      check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"),
-      check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+      check_filter!(Pleroma.Upload.Filters.Exiftool, "exiftool"),
+      check_filter!(Pleroma.Upload.Filters.Mogrify, "mogrify"),
+      check_filter!(Pleroma.Upload.Filters.Mogrifun, "mogrify")
     ]
 
     preview_proxy_commands_status =
@@ -213,7 +216,7 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_repo_pool_size!(result), do: result
 
-  defp check_filter(filter, command_required) do
+  defp check_filter!(filter, command_required) do
     filters = Config.get([Pleroma.Upload, :filters])
 
     if filter in filters and not Pleroma.Utils.command_available?(command_required) do
@@ -227,4 +230,32 @@ defmodule Pleroma.ApplicationRequirements do
       true
     end
   end
+
+  defp check_otp_version!(:ok, Tesla.Adapter.Gun) do
+    if version = Pleroma.OTPVersion.version() do
+      [major, minor] =
+        version
+        |> String.split(".")
+        |> Enum.map(&String.to_integer/1)
+        |> Enum.take(2)
+
+      if (major == 22 and minor < 2) or major < 22 do
+        Logger.error("
+            !!!OTP VERSION ERROR!!!
+            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
+            ")
+        {:error, "OTP version error"}
+      else
+        :ok
+      end
+    else
+      Logger.error("
+          !!!OTP VERSION ERROR!!!
+          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+          ")
+      {:error, "OTP version error"}
+    end
+  end
+
+  defp check_otp_version!(result, _), do: result
 end

lib/pleroma/application/start_up_dependencies.ex

@@ -0,0 +1,186 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.StartUpDependencies do
+  @moduledoc """
+  Starts common and config-dependent Pleroma dependencies.
+  """
+
+  alias Pleroma.Config
+  alias Pleroma.Web.Endpoint
+
+  require Cachex.Spec
+  require Logger
+
+  @type config_path() :: {atom(), atom()} | {atom(), atom(), [atom()]}
+  @type relation() :: {config_path(), module()}
+
+  @spec start_all(Pleroma.Application.env()) ::
+          :ok | {:error, {:already_started, pid()} | :max_children | term()}
+  def start_all(env) do
+    with :ok <- start_common_deps(env),
+         :ok <- start_config_dependent_deps(env) do
+      :ok
+    end
+  end
+
+  @spec adapter_module() :: module()
+  def adapter_module do
+    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+      Pleroma.Gun.GunSupervisor
+    else
+      Pleroma.HTTP.HackneySupervisor
+    end
+  end
+
+  @spec spec(module()) :: module() | {module(), keyword()}
+  def spec(Oban), do: {Oban, Config.get(Oban)}
+
+  def spec(Pleroma.Web.StreamerRegistry) do
+    {Registry,
+     [
+       name: Pleroma.Web.Streamer.registry(),
+       keys: :duplicate,
+       partitions: System.schedulers_online()
+     ]}
+  end
+
+  def spec(child), do: child
+
+  @spec cachex_spec({String.t(), keyword()}) :: :supervisor.child_spec()
+  def cachex_spec({type, opts}) do
+    %{
+      id: String.to_atom("cachex_" <> type),
+      start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
+      type: :worker
+    }
+  end
+
+  defp start_common_deps(env) do
+    fun = fn child ->
+      DynamicSupervisor.start_child(Pleroma.Application.dynamic_supervisor(), spec(child))
+    end
+
+    [
+      Pleroma.Emoji,
+      Pleroma.Stats,
+      Pleroma.JobQueueMonitor,
+      {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
+      %{
+        id: :web_push_init,
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+        restart: :temporary
+      }
+    ]
+    |> add_cachex_deps()
+    |> maybe_add_init_internal_fetch_actor_task(env)
+    |> maybe_add_background_migrator(env)
+    |> start_while(fun)
+  end
+
+  defp start_config_dependent_deps(env) do
+    fun = fn child -> Pleroma.Application.ConfigDependentDeps.start_dependency(child) end
+
+    [
+      Pleroma.Web.Plugs.RateLimiter.Supervisor,
+      Oban,
+      Endpoint,
+      Pleroma.Gopher.Server
+    ]
+    |> add_http_children(env)
+    |> maybe_add_streamer(env)
+    |> maybe_add_chat_child()
+    |> start_while(fun)
+  end
+
+  defp start_while(deps, fun) do
+    Enum.reduce_while(deps, :ok, fn child, acc ->
+      case fun.(child) do
+        {:ok, _} ->
+          {:cont, acc}
+
+        # consider this behavior is normal
+        :ignore ->
+          Logger.info("#{inspect(child)} is ignored.")
+          {:cont, acc}
+
+        error ->
+          Logger.error("Child #{inspect(child)} can't be started. #{inspect(error)}")
+          {:halt, error}
+      end
+    end)
+  end
+
+  @spec cachex_deps() :: [tuple()]
+  def cachex_deps do
+    captcha_clean_up_interval =
+      [Pleroma.Captcha, :seconds_valid]
+      |> Config.get!()
+      |> :timer.seconds()
+
+    [
+      {"used_captcha", expiration: Cachex.Spec.expiration(interval: captcha_clean_up_interval)},
+      {"user", expiration: cachex_expiration(25_000, 1000), limit: 2500},
+      {"object", expiration: cachex_expiration(25_000, 1000), limit: 2500},
+      {"rich_media",
+       expiration: Cachex.Spec.expiration(default: :timer.minutes(120)), limit: 5000},
+      {"scrubber", limit: 2500},
+      {"idempotency", expiration: cachex_expiration(21_600, 60), limit: 2500},
+      {"web_resp", limit: 2500},
+      {"emoji_packs", expiration: cachex_expiration(300, 60), limit: 10},
+      {"failed_proxy_url", limit: 2500},
+      {"banned_urls",
+       expiration: Cachex.Spec.expiration(default: :timer.hours(24 * 30)), limit: 5_000},
+      {"chat_message_id_idempotency_key",
+       expiration: cachex_expiration(:timer.minutes(2), :timer.seconds(60)), limit: 500_000}
+    ]
+  end
+
+  defp add_cachex_deps(application_deps) do
+    cachex_deps()
+    |> Enum.reduce(application_deps, fn cachex_init_args, acc ->
+      [cachex_spec(cachex_init_args) | acc]
+    end)
+  end
+
+  defp cachex_expiration(default, interval) do
+    Cachex.Spec.expiration(default: :timer.seconds(default), interval: :timer.seconds(interval))
+  end
+
+  defp maybe_add_init_internal_fetch_actor_task(children, :test), do: children
+
+  defp maybe_add_init_internal_fetch_actor_task(children, _) do
+    [
+      %{
+        id: :internal_fetch_init,
+        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
+        restart: :temporary
+      }
+      | children
+    ]
+  end
+
+  defp maybe_add_streamer(children, env) when env in [:test, :benchmark], do: children
+  defp maybe_add_streamer(children, _), do: [Pleroma.Web.Streamer.registry() | children]
+
+  defp maybe_add_background_migrator(children, env) when env in [:test, :benchmark], do: children
+
+  defp maybe_add_background_migrator(children, _) do
+    [Pleroma.Migrators.HashtagsTableMigrator | children]
+  end
+
+  defp add_http_children(children, :test) do
+    [Pleroma.HTTP.HackneySupervisor, Pleroma.Gun.GunSupervisor | children]
+  end
+
+  defp add_http_children(children, _), do: [adapter_module() | children]
+
+  defp maybe_add_chat_child(children) do
+    if Config.get([:chat, :enabled]) do
+      [Pleroma.Application.ChatSupervisor | children]
+    else
+      children
+    end
+  end
+end

lib/pleroma/config/deprecation_warnings.ex

@@ -41,7 +41,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
          :ok <- check_gun_pool_options(),
          :ok <- check_activity_expiration_config(),
          :ok <- check_remote_ip_plug_name(),
-         :ok <- check_uploders_s3_public_endpoint() do
+         :ok <- check_uploders_s3_public_endpoint(),
+         :ok <- check_oban_config() do
       :ok
     else
       _ ->
@@ -79,7 +80,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     move_namespace_and_warn(@mrf_config_map, warning_preface)
   end
 
-  @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil
+  @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | :error
   def move_namespace_and_warn(config_map, warning_preface) do
     warning =
       Enum.reduce(config_map, "", fn
@@ -102,7 +103,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
-  @spec check_media_proxy_whitelist_config() :: :ok | nil
+  @spec check_media_proxy_whitelist_config() :: :ok | :error
   def check_media_proxy_whitelist_config do
     whitelist = Config.get([:media_proxy, :whitelist])
 
@@ -163,7 +164,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
-  @spec check_activity_expiration_config() :: :ok | nil
+  @spec check_activity_expiration_config() :: :ok | :error
   def check_activity_expiration_config do
     warning_preface = """
     !!!DEPRECATION WARNING!!!
@@ -215,4 +216,41 @@ defmodule Pleroma.Config.DeprecationWarnings do
       :ok
     end
   end
+
+  @spec check_oban_config() :: :ok | :error
+  def check_oban_config do
+    oban_config = Config.get(Oban)
+
+    {crontab, changed?} =
+      [
+        Pleroma.Workers.Cron.StatsWorker,
+        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
+        Pleroma.Workers.Cron.ClearOauthTokenWorker
+      ]
+      |> Enum.reduce({oban_config[:crontab], false}, fn removed_worker, {acc, changed?} ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          {List.delete(acc, setting), true}
+        else
+          _ -> {acc, changed?}
+        end
+      end)
+
+    if changed? do
+      Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+      :error
+    else
+      :ok
+    end
+  end
 end

lib/pleroma/config/loader.ex

@@ -3,57 +3,76 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Loader do
+  @reject_groups [
+    :postgrex,
+    :tesla,
+    :phoenix,
+    :tzdata,
+    :http_signatures,
+    :web_push_encryption,
+    :floki,
+    :pbkdf2_elixir
+  ]
+
   @reject_keys [
     Pleroma.Repo,
     Pleroma.Web.Endpoint,
+    Pleroma.InstallerWeb.Endpoint,
     :env,
     :configurable_from_database,
     :database,
-    :swarm
-  ]
-
-  @reject_groups [
-    :postgrex,
-    :tesla
+    :ecto_repos,
+    Pleroma.Gun,
+    Pleroma.ReverseProxy.Client,
+    Pleroma.Web.Auth.Authenticator
   ]
 
   if Code.ensure_loaded?(Config.Reader) do
     @reader Config.Reader
-
-    def read(path), do: @reader.read!(path)
   else
     # support for Elixir less than 1.9
     @reader Mix.Config
-    def read(path) do
-      path
-      |> @reader.eval!()
-      |> elem(0)
-    end
   end
 
-  @spec read(Path.t()) :: keyword()
+  @spec read!(Path.t()) :: keyword()
+  def read!(path), do: @reader.read!(path)
 
   @spec merge(keyword(), keyword()) :: keyword()
   def merge(c1, c2), do: @reader.merge(c1, c2)
 
   @spec default_config() :: keyword()
   def default_config do
-    "config/config.exs"
-    |> read()
-    |> filter()
+    config =
+      "config/config.exs"
+      |> read!()
+      |> filter()
+
+    logger_config = Application.get_all_env(:logger)
+
+    merge(config, logger: logger_config)
   end
 
-  defp filter(configs) do
+  @spec filter(keyword()) :: keyword()
+  def filter(configs) do
     configs
-    |> Keyword.keys()
-    |> Enum.reduce([], &Keyword.put(&2, &1, filter_group(&1, configs)))
-  end
+    |> Enum.reduce([], fn
+      {group, _settings}, group_acc when group in @reject_groups ->
+        group_acc
 
-  @spec filter_group(atom(), keyword()) :: keyword()
-  def filter_group(group, configs) do
-    Enum.reject(configs[group], fn {key, _v} ->
-      key in @reject_keys or group in @reject_groups or
-        (group == :phoenix and key == :serve_endpoints)
+      {group, settings}, group_acc ->
+        filtered_settings =
+          Enum.reduce(settings, [], fn
+            {key, _value}, settings_acc when key in @reject_keys ->
+              settings_acc
+
+            {key, _value}, settings_acc when group == :phoenix and key == :serve_endpoint ->
+              settings_acc
+
+            {key, value}, settings_acc ->
+              Keyword.put(settings_acc, key, value)
+          end)
+
+        Keyword.put(group_acc, group, filtered_settings)
     end)
   end
 end

lib/pleroma/config/oban.ex

@@ -1,38 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Config.Oban do
-  require Logger
-
-  def warn do
-    oban_config = Pleroma.Config.get(Oban)
-
-    crontab =
-      [
-        Pleroma.Workers.Cron.StatsWorker,
-        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
-        Pleroma.Workers.Cron.ClearOauthTokenWorker
-      ]
-      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
-        with acc when is_list(acc) <- acc,
-             setting when is_tuple(setting) <-
-               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
-          """
-          !!!OBAN CONFIG WARNING!!!
-          You are using old workers in Oban crontab settings, which were removed.
-          Please, remove setting from crontab in your config file (prod.secret.exs): #{
-            inspect(setting)
-          }
-          """
-          |> Logger.warn()
-
-          List.delete(acc, setting)
-        else
-          _ -> acc
-        end
-      end)
-
-    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
-  end
-end

lib/pleroma/config/transfer_task.ex

@@ -1,201 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Config.TransferTask do
-  use Task
-
-  alias Pleroma.Config
-  alias Pleroma.ConfigDB
-  alias Pleroma.Repo
-
-  require Logger
-
-  @type env() :: :test | :benchmark | :dev | :prod
-
-  @reboot_time_keys [
-    {:pleroma, :hackney_pools},
-    {:pleroma, :chat},
-    {:pleroma, Oban},
-    {:pleroma, :rate_limit},
-    {:pleroma, :markup},
-    {:pleroma, :streamer},
-    {:pleroma, :pools},
-    {:pleroma, :connections_pool}
-  ]
-
-  @reboot_time_subkeys [
-    {:pleroma, Pleroma.Captcha, [:seconds_valid]},
-    {:pleroma, Pleroma.Upload, [:proxy_remote]},
-    {:pleroma, :instance, [:upload_limit]},
-    {:pleroma, :gopher, [:enabled]}
-  ]
-
-  def start_link(restart_pleroma? \\ true) do
-    load_and_update_env([], restart_pleroma?)
-    if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
-    :ignore
-  end
-
-  @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
-  def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
-    with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
-      # We need to restart applications for loaded settings take effect
-
-      {logger, other} =
-        (Repo.all(ConfigDB) ++ deleted_settings)
-        |> Enum.map(&merge_with_default/1)
-        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
-
-      logger
-      |> Enum.sort()
-      |> Enum.each(&configure/1)
-
-      started_applications = Application.started_applications()
-
-      # TODO: some problem with prometheus after restart!
-      reject = [nil, :prometheus, :postgrex]
-
-      reject =
-        if restart_pleroma? do
-          reject
-        else
-          [:pleroma | reject]
-        end
-
-      other
-      |> Enum.map(&update/1)
-      |> Enum.uniq()
-      |> Enum.reject(&(&1 in reject))
-      |> maybe_set_pleroma_last()
-      |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
-
-      :ok
-    else
-      {:configurable, false} -> Restarter.Pleroma.rebooted()
-    end
-  end
-
-  defp maybe_set_pleroma_last(apps) do
-    # to be ensured that pleroma will be restarted last
-    if :pleroma in apps do
-      apps
-      |> List.delete(:pleroma)
-      |> List.insert_at(-1, :pleroma)
-    else
-      Restarter.Pleroma.rebooted()
-      apps
-    end
-  end
-
-  defp merge_with_default(%{group: group, key: key, value: value} = setting) do
-    default = Config.Holder.default_config(group, key)
-
-    merged =
-      cond do
-        Ecto.get_meta(setting, :state) == :deleted -> default
-        can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
-        true -> value
-      end
-
-    {group, key, value, merged}
-  end
-
-  # change logger configuration in runtime, without restart
-  defp configure({:quack, key, _, merged}) do
-    Logger.configure_backend(Quack.Logger, [{key, merged}])
-    :ok = update_env(:quack, key, merged)
-  end
-
-  defp configure({_, :backends, _, merged}) do
-    # removing current backends
-    Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
-
-    Enum.each(merged, &Logger.add_backend/1)
-
-    :ok = update_env(:logger, :backends, merged)
-  end
-
-  defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
-    merged =
-      if key == :console do
-        put_in(merged[:format], merged[:format] <> "\n")
-      else
-        merged
-      end
-
-    backend =
-      if key == :ex_syslogger,
-        do: {ExSyslogger, :ex_syslogger},
-        else: key
-
-    Logger.configure_backend(backend, merged)
-    :ok = update_env(:logger, key, merged)
-  end
-
-  defp configure({_, key, _, merged}) do
-    Logger.configure([{key, merged}])
-    :ok = update_env(:logger, key, merged)
-  end
-
-  defp update({group, key, value, merged}) do
-    try do
-      :ok = update_env(group, key, merged)
-
-      if group != :pleroma or pleroma_need_restart?(group, key, value), do: group
-    rescue
-      error ->
-        error_msg =
-          "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{
-            inspect(value)
-          } error: #{inspect(error)}"
-
-        Logger.warn(error_msg)
-
-        nil
-    end
-  end
-
-  defp update_env(group, key, nil), do: Application.delete_env(group, key)
-  defp update_env(group, key, value), do: Application.put_env(group, key, value)
-
-  @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
-  def pleroma_need_restart?(group, key, value) do
-    group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
-  end
-
-  defp group_and_key_need_reboot?(group, key) do
-    Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end)
-  end
-
-  defp group_and_subkey_need_reboot?(group, key, value) do
-    Keyword.keyword?(value) and
-      Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} ->
-        g == group and k == key and
-          Enum.any?(Keyword.keys(value), &(&1 in subkeys))
-      end)
-  end
-
-  defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
-
-  defp restart(started_applications, app, _) do
-    with {^app, _, _} <- List.keyfind(started_applications, app, 0),
-         :ok <- Application.stop(app) do
-      :ok = Application.start(app)
-    else
-      nil ->
-        Logger.warn("#{app} is not started.")
-
-      error ->
-        error
-        |> inspect()
-        |> Logger.warn()
-    end
-  end
-
-  defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
-    Keyword.keyword?(val1) and Keyword.keyword?(val2)
-  end
-
-  defp can_be_merged?(_val1, _val2), do: false
-end

lib/pleroma/config_db.ex

@@ -31,6 +31,9 @@ defmodule Pleroma.ConfigDB do
     timestamps()
   end
 
+  @spec all() :: [t()]
+  def all, do: Repo.all(ConfigDB)
+
   @spec get_all_as_keyword() :: keyword()
   def get_all_as_keyword do
     ConfigDB
@@ -106,6 +109,27 @@ defmodule Pleroma.ConfigDB do
     |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1]))
   end
 
+  @spec merge_with_default(t(), keyword()) :: {t(), any()}
+  def merge_with_default(%ConfigDB{} = change, defaults) do
+    %{group: group, key: key, value: value} = change
+    default = defaults[group][key]
+
+    merged =
+      cond do
+        Ecto.get_meta(change, :state) == :deleted -> default
+        can_be_merged?(default, value) -> merge_group(group, key, default, value)
+        true -> value
+      end
+
+    {change, merged}
+  end
+
+  defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
+    Keyword.keyword?(val1) and Keyword.keyword?(val2)
+  end
+
+  defp can_be_merged?(_val1, _val2), do: false
+
   defp to_mapset(keyword) do
     keyword
     |> Keyword.keys()

lib/pleroma/docs/json.ex

@@ -5,7 +5,7 @@
 defmodule Pleroma.Docs.JSON do
   @behaviour Pleroma.Docs.Generator
   @external_resource "config/description.exs"
-  @raw_config Pleroma.Config.Loader.read("config/description.exs")
+  @raw_config Pleroma.Config.Loader.read!("config/description.exs")
   @raw_descriptions @raw_config[:pleroma][:config_description]
   @term __MODULE__.Compiled
 

lib/pleroma/gopher/server.ex

@@ -12,13 +12,14 @@ defmodule Pleroma.Gopher.Server do
     port = Keyword.get(config, :port, 1234)
 
     if Keyword.get(config, :enabled, false) do
-      GenServer.start_link(__MODULE__, [ip, port], [])
+      GenServer.start_link(__MODULE__, [ip, port])
     else
       Logger.info("Gopher server disabled")
       :ignore
     end
   end
 
+  @impl true
   def init([ip, port]) do
     Logger.info("Starting gopher server on #{port}")
 
@@ -31,8 +32,14 @@ defmodule Pleroma.Gopher.Server do
       []
     )
 
+    Process.flag(:trap_exit, true)
     {:ok, %{ip: ip, port: port}}
   end
+
+  @impl true
+  def terminate(_reason, _state) do
+    :ranch.stop_listener(:gopher)
+  end
 end
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do

lib/pleroma/gun/gun_supervisor.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.GunSupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_args)
+  end
+
+  def init(_) do
+    children =
+      Pleroma.Gun.ConnectionPool.children() ++
+        [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end

lib/pleroma/http/hackney_supervisor.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.HackneySupervisor do
+  use Supervisor
+
+  def start_link(_) do
+    Supervisor.start_link(__MODULE__, :no_arg)
+  end
+
+  def init(_) do
+    pools = [:federation, :media]
+
+    pools =
+      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    children =
+      for pool <- pools do
+        options = Pleroma.Config.get([:hackney_pools, pool])
+
+        :hackney_pool.child_spec(pool, options)
+      end
+
+    Supervisor.init(children, strategy: :one_for_one)
+  end
+end

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -392,14 +392,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def restart(conn, _params) do
     with :ok <- configurable_from_database() do
-      Restarter.Pleroma.restart(Config.get(:env), 50)
+      Task.start(Pleroma.Application.ConfigDependentDeps, :restart_dependencies, [])
 
       json(conn, %{})
     end
   end
 
   def need_reboot(conn, _params) do
-    json(conn, %{need_reboot: Restarter.Pleroma.need_reboot?()})
+    json(conn, %{need_reboot: Pleroma.Application.ConfigDependentDeps.need_reboot?()})
   end
 
   defp configurable_from_database do

lib/pleroma/web/admin_api/controllers/config_controller.ex

@@ -34,7 +34,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
 
       render(conn, "index.json", %{
         configs: configs,
-        need_reboot: Restarter.Pleroma.need_reboot?()
+        need_reboot: Pleroma.Application.ConfigDependentDeps.need_reboot?()
       })
     end
   end
@@ -75,7 +75,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
 
       render(conn, "index.json", %{
         configs: merged,
-        need_reboot: Restarter.Pleroma.need_reboot?()
+        need_reboot: Pleroma.Application.ConfigDependentDeps.need_reboot?()
       })
     end
   end
@@ -93,27 +93,15 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
             ConfigDB.update_or_create(%{group: group, key: key, value: value})
         end)
         |> Enum.reject(fn {result, _} -> result == :error end)
-
-      {deleted, updated} =
-        results
         |> Enum.map(fn {:ok, %{key: key, value: value} = config} ->
           Map.put(config, :db, ConfigDB.get_db_keys(value, key))
         end)
-        |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted))
 
-      Config.TransferTask.load_and_update_env(deleted, false)
-
-      if not Restarter.Pleroma.need_reboot?() do
-        changed_reboot_settings? =
-          (updated ++ deleted)
-          |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value))
-
-        if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
-      end
+      Pleroma.Application.Environment.update(results, only_update: true)
 
       render(conn, "index.json", %{
-        configs: updated,
-        need_reboot: Restarter.Pleroma.need_reboot?()
+        configs: Enum.reject(results, &(Ecto.get_meta(&1, :state) == :deleted)),
+        need_reboot: Pleroma.Application.ConfigDependentDeps.need_reboot?()
       })
     end
   end

mix.exs

@@ -192,7 +192,6 @@ defmodule Pleroma.Mixfile do
       {:captcha,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git",
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
-      {:restarter, path: "./restarter"},
       {:majic,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git",
        ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"},

priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs

@@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.MoveActivityExpirationsToOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
-    Pleroma.Config.Oban.warn()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
 
     Application.ensure_all_started(:oban)
 

priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs

@@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.MoveTokensExpirationIntoOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
-    Pleroma.Config.Oban.warn()
+    Pleroma.Config.DeprecationWarnings.check_oban_config()
 
     Application.ensure_all_started(:oban)
 

restarter/lib/pleroma.ex

@@ -1,94 +0,0 @@
-defmodule Restarter.Pleroma do
-  use GenServer
-
-  require Logger
-
-  @init_state %{need_reboot: false, rebooted: false, after_boot: false}
-
-  def start_link(_) do
-    GenServer.start_link(__MODULE__, [], name: __MODULE__)
-  end
-
-  def init(_), do: {:ok, @init_state}
-
-  def rebooted? do
-    GenServer.call(__MODULE__, :rebooted?)
-  end
-
-  def rebooted do
-    GenServer.cast(__MODULE__, :rebooted)
-  end
-
-  def need_reboot? do
-    GenServer.call(__MODULE__, :need_reboot?)
-  end
-
-  def need_reboot do
-    GenServer.cast(__MODULE__, :need_reboot)
-  end
-
-  def refresh do
-    GenServer.cast(__MODULE__, :refresh)
-  end
-
-  def restart(env, delay) do
-    GenServer.cast(__MODULE__, {:restart, env, delay})
-  end
-
-  def restart_after_boot(env) do
-    GenServer.cast(__MODULE__, {:after_boot, env})
-  end
-
-  def handle_call(:rebooted?, _from, state) do
-    {:reply, state[:rebooted], state}
-  end
-
-  def handle_call(:need_reboot?, _from, state) do
-    {:reply, state[:need_reboot], state}
-  end
-
-  def handle_cast(:rebooted, state) do
-    {:noreply, Map.put(state, :rebooted, true)}
-  end
-
-  def handle_cast(:need_reboot, %{need_reboot: true} = state), do: {:noreply, state}
-
-  def handle_cast(:need_reboot, state) do
-    {:noreply, Map.put(state, :need_reboot, true)}
-  end
-
-  def handle_cast(:refresh, _state) do
-    {:noreply, @init_state}
-  end
-
-  def handle_cast({:restart, :test, _}, state) do
-    Logger.debug("pleroma manually restarted")
-    {:noreply, Map.put(state, :need_reboot, false)}
-  end
-
-  def handle_cast({:restart, _, delay}, state) do
-    Process.sleep(delay)
-    do_restart(:pleroma)
-    {:noreply, Map.put(state, :need_reboot, false)}
-  end
-
-  def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state}
-
-  def handle_cast({:after_boot, :test}, state) do
-    Logger.debug("pleroma restarted after boot")
-    state = %{state | after_boot: true, rebooted: true}
-    {:noreply, state}
-  end
-
-  def handle_cast({:after_boot, _}, state) do
-    do_restart(:pleroma)
-    state = %{state | after_boot: true, rebooted: true}
-    {:noreply, state}
-  end
-
-  defp do_restart(app) do
-    :ok = Application.ensure_started(app)
-    :ok = Application.stop(app)
-    :ok = Application.start(app)
-  end
-end

restarter/lib/restarter.ex

@@ -1,8 +0,0 @@
-defmodule Restarter do
-  use Application
-
-  def start(_, _) do
-    opts = [strategy: :one_for_one, name: Restarter.Supervisor]
-    Supervisor.start_link([Restarter.Pleroma], opts)
-  end
-end

restarter/mix.exs

@@ -1,21 +0,0 @@
-defmodule Restarter.MixProject do
-  use Mix.Project
-
-  def project do
-    [
-      app: :restarter,
-      version: "0.1.0",
-      elixir: "~> 1.8",
-      start_permanent: Mix.env() == :prod,
-      deps: deps()
-    ]
-  end
-
-  def application do
-    [
-      mod: {Restarter, []}
-    ]
-  end
-
-  defp deps, do: []
-end

test/fixtures/config/temp.secret.exs

@@ -12,6 +12,30 @@ config :quack, level: :info
 
 config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox
 
-config :postgrex, :json_library, Poison
+config :pleroma, Pleroma.Web.Endpoint, key: :val
+
+config :pleroma, Pleroma.InstallerWeb.Endpoint, key: :val
+
+config :pleroma, env: :test
 
 config :pleroma, :database, rum_enabled: true
+
+config :pleroma, configurable_from_database: false
+
+config :pleroma, ecto_repos: [Pleroma.Repo]
+
+config :pleroma, Pleroma.Gun, Pleroma.GunMock
+
+config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.Client
+
+config :postgrex, :json_library, Poison
+
+config :tesla, adapter: Tesla.Mock
+
+config :tzdata, http_client: Pleroma.HTTP
+
+config :http_signatures, key: :val
+
+config :web_push_encryption, key: :val
+
+config :floki, key: :val

test/pleroma/application/config_dependent_deps_test.exs

@@ -0,0 +1,149 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.ConfigDependentDepsTest do
+  use ExUnit.Case
+
+  alias Pleroma.Application.ConfigDependentDeps
+
+  setup do
+    {:ok, _} =
+      DynamicSupervisor.start_link(
+        strategy: :one_for_one,
+        name: Pleroma.Application.DynamicSupervisorTest
+      )
+
+    {:ok, pid} =
+      Pleroma.Application.ConfigDependentDeps.start_link(
+        dynamic_supervisor: Pleroma.Application.DynamicSupervisorTest,
+        name: Pleroma.Application.ConfigDependentDepsTesting,
+        relations: [
+          {{:pleroma, :dummy_module1}, Pleroma.DummyModule1},
+          {{:pleroma, :dummy_module2}, Pleroma.DummyModule2},
+          {:dummy_group1, :dummy_group1},
+          {:ex_aws, :ex_aws},
+          {:not_started_app, :not_started_app}
+        ]
+      )
+
+    [pid: pid]
+  end
+
+  test "start_dependency/2", %{pid: pid} do
+    {:ok, pid} = ConfigDependentDeps.start_dependency(Pleroma.DummyModule1, pid)
+    assert Process.alive?(pid)
+  end
+
+  describe "need_reboot?/1" do
+    test "apps and paths", %{pid: pid} do
+      changes = [
+        %Pleroma.ConfigDB{group: :dummy_group1},
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module1}
+      ]
+
+      assert ConfigDependentDeps.save_config_paths_for_restart(changes, pid) == [
+               {:pleroma, :dummy_module1},
+               :dummy_group1
+             ]
+
+      assert ConfigDependentDeps.need_reboot?(pid)
+    end
+
+    test "app and path are not duplicated", %{pid: pid} do
+      changes = [
+        %Pleroma.ConfigDB{group: :dummy_group1},
+        %Pleroma.ConfigDB{group: :dummy_group1},
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module1},
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module1}
+      ]
+
+      assert ConfigDependentDeps.save_config_paths_for_restart(changes, pid) == [
+               {:pleroma, :dummy_module1},
+               :dummy_group1
+             ]
+
+      assert ConfigDependentDeps.need_reboot?(pid)
+    end
+  end
+
+  describe "restart_dependencies/1" do
+    test "started dependency", %{pid: pid} do
+      {:ok, dummy_pid} = ConfigDependentDeps.start_dependency(Pleroma.DummyModule1, pid)
+
+      changes = [
+        %Pleroma.ConfigDB{group: :ex_aws},
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module1}
+      ]
+
+      assert ConfigDependentDeps.save_config_paths_for_restart(changes, pid) == [
+               {:pleroma, :dummy_module1},
+               :ex_aws
+             ]
+
+      assert :ok == ConfigDependentDeps.restart_dependencies(pid)
+
+      restarted = Process.whereis(Pleroma.DummyModule1)
+
+      refute dummy_pid == restarted
+    end
+
+    test "not started process and app", %{pid: pid} do
+      changes = [
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module1},
+        %Pleroma.ConfigDB{group: :not_started_app}
+      ]
+
+      assert ConfigDependentDeps.save_config_paths_for_restart(changes, pid) == [
+               :not_started_app,
+               {:pleroma, :dummy_module1}
+             ]
+
+      assert :ok == ConfigDependentDeps.restart_dependencies(pid)
+
+      started = Process.whereis(Pleroma.DummyModule1)
+
+      assert Process.alive?(started)
+    end
+
+    test "ignored dependency", %{pid: pid} do
+      changes = [
+        %Pleroma.ConfigDB{group: :pleroma, key: :dummy_module2}
+      ]
+
+      assert ConfigDependentDeps.save_config_paths_for_restart(changes, pid) == [
+               {:pleroma, :dummy_module2}
+             ]
+
+      assert :ok == ConfigDependentDeps.restart_dependencies(pid)
+
+      refute Process.whereis(Pleroma.DummyModule2)
+    end
+  end
+
+  test "process goes down", %{pid: pid} do
+    {:ok, dummy_pid} = ConfigDependentDeps.start_dependency(Pleroma.DummyModule1, pid)
+
+    Process.exit(dummy_pid, :kill)
+
+    Process.sleep(10)
+    restarted = Process.whereis(Pleroma.DummyModule1)
+    refute restarted == dummy_pid
+  end
+end
+
+defmodule Pleroma.DummyModule1 do
+  use Agent
+
+  def start_link(_) do
+    Agent.start_link(fn -> nil end, name: __MODULE__)
+  end
+end
+
+defmodule Pleroma.DummyModule2 do
+  use Agent
+
+  def start_link(_) do
+    :ignore
+  end
+end

test/pleroma/application/environment_test.exs

@@ -0,0 +1,243 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Application.EnvironmentTest do
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  alias Pleroma.Application.Environment
+
+  setup do: clear_config(:configurable_from_database, true)
+
+  describe "load_from_db_and_update/0" do
+    test "transfer config values from db to env" do
+      refute Application.get_env(:pleroma, :test_key)
+      refute Application.get_env(:idna, :test_key)
+      refute Application.get_env(:quack, :test_key)
+      refute Application.get_env(:postgrex, :test_key)
+      initial = Application.get_env(:logger, :level)
+
+      insert(:config, key: :test_key, value: [live: 2, com: 3])
+      insert(:config, group: :idna, key: :test_key, value: [live: 15, com: 35])
+
+      insert(:config,
+        group: :quack,
+        key: :test_key,
+        value: [key1: :test_value1, key2: :test_value2]
+      )
+
+      insert(:config, group: :logger, key: :level, value: :debug)
+
+      Environment.load_from_db_and_update()
+
+      assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
+      assert Application.get_env(:idna, :test_key) == [live: 15, com: 35]
+      assert Application.get_env(:quack, :test_key) == [key1: :test_value1, key2: :test_value2]
+      assert Application.get_env(:logger, :level) == :debug
+
+      on_exit(fn ->
+        Application.delete_env(:pleroma, :test_key)
+        Application.delete_env(:idna, :test_key)
+        Application.delete_env(:quack, :test_key)
+        Application.delete_env(:postgrex, :test_key)
+        Application.put_env(:logger, :level, initial)
+      end)
+    end
+
+    test "transfer config values for 1 group and some keys" do
+      quack_env = Application.get_all_env(:quack)
+
+      insert(:config, group: :quack, key: :level, value: :info)
+      insert(:config, group: :quack, key: :meta, value: [:none])
+
+      Environment.load_from_db_and_update()
+
+      assert Application.get_env(:quack, :level) == :info
+      assert Application.get_env(:quack, :meta) == [:none]
+      default = Pleroma.Config.Holder.default_config(:quack, :webhook_url)
+      assert Application.get_env(:quack, :webhook_url) == default
+
+      on_exit(fn ->
+        Application.put_all_env(quack: quack_env)
+      end)
+    end
+
+    test "transfer config values with full subkey update" do
+      clear_config(:emoji)
+      clear_config(:assets)
+
+      insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
+      insert(:config, key: :assets, value: [mascots: [a: 1, b: 2]])
+
+      Environment.load_from_db_and_update()
+
+      emoji_env = Application.get_env(:pleroma, :emoji)
+      assert emoji_env[:groups] == [a: 1, b: 2]
+      assets_env = Application.get_env(:pleroma, :assets)
+      assert assets_env[:mascots] == [a: 1, b: 2]
+    end
+  end
+
+  describe "update/2 :ex_syslogger" do
+    setup do
+      initial = Application.get_env(:logger, :ex_syslogger)
+
+      config =
+        insert(:config,
+          group: :logger,
+          key: :ex_syslogger,
+          value: [
+            level: :warn,
+            ident: "pleroma",
+            format: "$metadata[$level] $message",
+            metadata: [:request_id, :key]
+          ]
+        )
+
+      on_exit(fn -> Application.put_env(:logger, :ex_syslogger, initial) end)
+      [config: config, initial: initial]
+    end
+
+    test "changing", %{config: config} do
+      assert Environment.update([config]) == :ok
+
+      env = Application.get_env(:logger, :ex_syslogger)
+      assert env[:level] == :warn
+      assert env[:metadata] == [:request_id, :key]
+    end
+
+    test "deletion", %{config: config, initial: initial} do
+      assert Environment.update([config]) == :ok
+
+      {:ok, config} = Pleroma.ConfigDB.delete(config)
+      assert Environment.update([config]) == :ok
+
+      env = Application.get_env(:logger, :ex_syslogger)
+
+      assert env == initial
+    end
+  end
+
+  describe "update/2 :console" do
+    setup do
+      initial = Application.get_env(:logger, :console)
+
+      config =
+        insert(:config,
+          group: :logger,
+          key: :console,
+          value: [
+            level: :info,
+            format: "$time $metadata[$level]",
+            metadata: [:request_id, :key]
+          ]
+        )
+
+      on_exit(fn -> Application.put_env(:logger, :console, initial) end)
+      [config: config, initial: initial]
+    end
+
+    test "change", %{config: config} do
+      assert Environment.update([config]) == :ok
+      env = Application.get_env(:logger, :console)
+      assert env[:level] == :info
+      assert env[:format] == "$time $metadata[$level]"
+      assert env[:metadata] == [:request_id, :key]
+    end
+
+    test "deletion", %{config: config, initial: initial} do
+      assert Environment.update([config]) == :ok
+      {:ok, config} = Pleroma.ConfigDB.delete(config)
+      assert Environment.update([config]) == :ok
+
+      env = Application.get_env(:logger, :console)
+      assert env == initial
+    end
+  end
+
+  describe "update/2 :backends" do
+    setup do
+      initial = Application.get_all_env(:logger)
+
+      config = insert(:config, group: :logger, key: :backends, value: [:console, :ex_syslogger])
+
+      on_exit(fn -> Application.put_all_env(logger: initial) end)
+
+      [config: config, initial: initial]
+    end
+
+    test "change", %{config: config} do
+      assert Environment.update([config]) == :ok
+      env = Application.get_all_env(:logger)
+      assert env[:backends] == [:console, :ex_syslogger]
+    end
+
+    test "deletion", %{config: config, initial: initial} do
+      assert Environment.update([config]) == :ok
+      {:ok, config} = Pleroma.ConfigDB.delete(config)
+      assert Environment.update([config])
+
+      env = Application.get_all_env(:logger)
+      assert env == initial
+    end
+  end
+
+  test "update/2 logger settings" do
+    initial = Application.get_all_env(:logger)
+
+    config1 =
+      insert(:config,
+        group: :logger,
+        key: :console,
+        value: [
+          level: :info,
+          format: "$time $metadata[$level]",
+          metadata: [:request_id, :key]
+        ]
+      )
+
+    config2 =
+      insert(:config,
+        group: :logger,
+        key: :ex_syslogger,
+        value: [
+          level: :warn,
+          ident: "pleroma",
+          format: "$metadata[$level] $message",
+          metadata: [:request_id, :key]
+        ]
+      )
+
+    config3 = insert(:config, group: :logger, key: :backends, value: [:console, :ex_syslogger])
+
+    on_exit(fn -> Application.put_all_env(logger: initial) end)
+
+    assert Environment.update([config1, config2, config3]) == :ok
+
+    env =
+      :logger
+      |> Application.get_all_env()
+      |> Keyword.take([:backends, :console, :ex_syslogger])
+
+    assert env[:console] == config1.value
+    assert env[:ex_syslogger] == config2.value
+    assert env[:backends] == config3.value
+  end
+
+  test "update/2 for change without key :cors_plug" do
+    initial = Application.get_all_env(:cors_plug)
+    config1 = insert(:config, group: :cors_plug, key: :max_age, value: 300)
+    config2 = insert(:config, group: :cors_plug, key: :methods, value: ["GET"])
+
+    assert Environment.update([config1, config2]) == :ok
+
+    env = Application.get_all_env(:cors_plug)
+
+    assert env[:max_age] == 300
+    assert env[:methods] == ["GET"]
+
+    on_exit(fn -> Application.put_all_env(cors_plug: initial) end)
+  end
+end

test/pleroma/application/requirements_test.exs

@@ -2,24 +2,24 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ApplicationRequirementsTest do
+defmodule Pleroma.Application.RequirementsTest do
   use Pleroma.DataCase
 
   import ExUnit.CaptureLog
   import Mock
 
-  alias Pleroma.ApplicationRequirements
-  alias Pleroma.Repo
+  alias Pleroma.Application.Requirements
+  alias Pleroma.Emails.Mailer
 
   describe "check_repo_pool_size!/1" do
     test "raises if the pool size is unexpected" do
       clear_config([Pleroma.Repo, :pool_size], 11)
       clear_config([:dangerzone, :override_repo_pool_size], false)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+      assert_raise Requirements.VerifyError,
                    "Repo.pool_size different than recommended value.",
                    fn ->
-                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     capture_log(&Requirements.verify!/0)
                    end
     end
 
@@ -27,27 +27,27 @@ defmodule Pleroma.ApplicationRequirementsTest do
       clear_config([Pleroma.Repo, :pool_size], 11)
       clear_config([:dangerzone, :override_repo_pool_size], true)
 
-      assert Pleroma.ApplicationRequirements.verify!() == :ok
+      assert Requirements.verify!() == :ok
     end
   end
 
   describe "check_welcome_message_config!/1" do
     setup do: clear_config([:welcome])
-    setup do: clear_config([Pleroma.Emails.Mailer])
+    setup do: clear_config([Mailer])
 
     test "raises if welcome email enabled but mail disabled" do
       clear_config([:welcome, :email, :enabled], true)
-      clear_config([Pleroma.Emails.Mailer, :enabled], false)
+      clear_config([Mailer, :enabled], false)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError, "The mail disabled.", fn ->
-        capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+      assert_raise Requirements.VerifyError, "The mail disabled.", fn ->
+        capture_log(&Requirements.verify!/0)
       end
     end
   end
 
   describe "check_confirmation_accounts!" do
     setup_with_mocks([
-      {Pleroma.ApplicationRequirements, [:passthrough],
+      {Requirements, [:passthrough],
        [
          check_migrations_applied!: fn _ -> :ok end
        ]}
@@ -59,32 +59,31 @@ defmodule Pleroma.ApplicationRequirementsTest do
 
     test "raises if account confirmation is required but mailer isn't enable" do
       clear_config([:instance, :account_activation_required], true)
-      clear_config([Pleroma.Emails.Mailer, :enabled], false)
+      clear_config([Mailer, :enabled], false)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+      assert_raise Requirements.VerifyError,
                    "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails.",
                    fn ->
-                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     capture_log(&Requirements.verify!/0)
                    end
     end
 
     test "doesn't do anything if account confirmation is disabled" do
       clear_config([:instance, :account_activation_required], false)
-      clear_config([Pleroma.Emails.Mailer, :enabled], false)
-      assert Pleroma.ApplicationRequirements.verify!() == :ok
+      clear_config([Mailer, :enabled], false)
+      assert Requirements.verify!() == :ok
     end
 
     test "doesn't do anything if account confirmation is required and mailer is enabled" do
       clear_config([:instance, :account_activation_required], true)
-      clear_config([Pleroma.Emails.Mailer, :enabled], true)
-      assert Pleroma.ApplicationRequirements.verify!() == :ok
+      clear_config([Mailer, :enabled], true)
+      assert Requirements.verify!() == :ok
     end
   end
 
   describe "check_rum!" do
     setup_with_mocks([
-      {Pleroma.ApplicationRequirements, [:passthrough],
-       [check_migrations_applied!: fn _ -> :ok end]}
+      {Requirements, [:passthrough], [check_migrations_applied!: fn _ -> :ok end]}
     ]) do
       :ok
     end
@@ -95,10 +94,10 @@ defmodule Pleroma.ApplicationRequirementsTest do
       clear_config([:database, :rum_enabled], true)
 
       with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
-        assert_raise ApplicationRequirements.VerifyError,
+        assert_raise Requirements.VerifyError,
                      "Unapplied RUM Migrations detected",
                      fn ->
-                       capture_log(&ApplicationRequirements.verify!/0)
+                       capture_log(&Requirements.verify!/0)
                      end
       end
     end
@@ -107,10 +106,10 @@ defmodule Pleroma.ApplicationRequirementsTest do
       clear_config([:database, :rum_enabled], false)
 
       with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
-        assert_raise ApplicationRequirements.VerifyError,
+        assert_raise Requirements.VerifyError,
                      "RUM Migrations detected",
                      fn ->
-                       capture_log(&ApplicationRequirements.verify!/0)
+                       capture_log(&Requirements.verify!/0)
                      end
       end
     end
@@ -119,7 +118,7 @@ defmodule Pleroma.ApplicationRequirementsTest do
       clear_config([:database, :rum_enabled], true)
 
       with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
-        assert ApplicationRequirements.verify!() == :ok
+        assert Requirements.verify!() == :ok
       end
     end
 
@@ -127,12 +126,12 @@ defmodule Pleroma.ApplicationRequirementsTest do
       clear_config([:database, :rum_enabled], false)
 
       with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
-        assert ApplicationRequirements.verify!() == :ok
+        assert Requirements.verify!() == :ok
       end
     end
   end
 
-  describe "check_migrations_applied!" do
+  describe "check_migrations_applied" do
     setup_with_mocks([
       {Ecto.Migrator, [],
        [
@@ -152,17 +151,17 @@ defmodule Pleroma.ApplicationRequirementsTest do
     setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
 
     test "raises if it detects unapplied migrations" do
-      assert_raise ApplicationRequirements.VerifyError,
+      assert_raise Requirements.VerifyError,
                    "Unapplied Migrations detected",
                    fn ->
-                     capture_log(&ApplicationRequirements.verify!/0)
+                     capture_log(&Requirements.verify!/0)
                    end
     end
 
     test "doesn't do anything if disabled" do
       clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
 
-      assert :ok == ApplicationRequirements.verify!()
+      assert :ok == Requirements.verify!()
     end
   end
 end

test/pleroma/config/loader_test.exs

@@ -8,22 +8,39 @@ defmodule Pleroma.Config.LoaderTest do
   alias Pleroma.Config.Loader
 
   test "read/1" do
-    config = Loader.read("test/fixtures/config/temp.secret.exs")
+    config = Loader.read!("test/fixtures/config/temp.secret.exs")
     assert config[:pleroma][:first_setting][:key] == "value"
     assert config[:pleroma][:first_setting][:key2] == [Pleroma.Repo]
     assert config[:quack][:level] == :info
   end
 
-  test "filter_group/2" do
-    assert Loader.filter_group(:pleroma,
-             pleroma: [
-               {Pleroma.Repo, [a: 1, b: 2]},
-               {Pleroma.Upload, [a: 1, b: 2]},
-               {Pleroma.Web.Endpoint, []},
-               env: :test,
-               configurable_from_database: true,
-               database: []
-             ]
-           ) == [{Pleroma.Upload, [a: 1, b: 2]}]
+  test "filter/1" do
+    config = Loader.read!("test/fixtures/config/temp.secret.exs")
+
+    filtered_config = Loader.filter(config)
+
+    refute filtered_config[:postgrex]
+    refute filtered_config[:tesla]
+    refute filtered_config[:phoenix]
+    refute filtered_config[:tz_data]
+    refute filtered_config[:http_signatures]
+    refute filtered_config[:web_push_encryption]
+    refute filtered_config[:floki]
+
+    refute filtered_config[:pleroma][Pleroma.Repo]
+    refute filtered_config[:pleroma][Pleroma.Web.Endpoint]
+    refute filtered_config[:pleroma][Pleroma.InstallerWeb.Endpoint]
+    refute filtered_config[:pleroma][:env]
+    refute filtered_config[:pleroma][:configurable_from_database]
+    refute filtered_config[:pleroma][:database]
+    refute filtered_config[:pleroma][:ecto_repos]
+    refute filtered_config[:pleroma][Pleroma.Gun]
+    refute filtered_config[:pleroma][Pleroma.ReverseProxy.Client]
+
+    assert config[:pleroma][:first_setting][:key] == "value"
+    assert config[:pleroma][:first_setting][:key2] == [Pleroma.Repo]
+    assert config[:quack][:level] == :info
+    assert config[:pleroma][:second_setting][:key] == "value2"
+    assert config[:pleroma][:second_setting][:key2] == ["Activity"]
   end
 end

test/pleroma/config/transfer_task_test.exs

@@ -1,120 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Config.TransferTaskTest do
-  use Pleroma.DataCase
-
-  import ExUnit.CaptureLog
-  import Pleroma.Factory
-
-  alias Pleroma.Config.TransferTask
-
-  setup do: clear_config(:configurable_from_database, true)
-
-  test "transfer config values from db to env" do
-    refute Application.get_env(:pleroma, :test_key)
-    refute Application.get_env(:idna, :test_key)
-    refute Application.get_env(:quack, :test_key)
-    refute Application.get_env(:postgrex, :test_key)
-    initial = Application.get_env(:logger, :level)
-
-    insert(:config, key: :test_key, value: [live: 2, com: 3])
-    insert(:config, group: :idna, key: :test_key, value: [live: 15, com: 35])
-    insert(:config, group: :quack, key: :test_key, value: [:test_value1, :test_value2])
-    insert(:config, group: :postgrex, key: :test_key, value: :value)
-    insert(:config, group: :logger, key: :level, value: :debug)
-
-    TransferTask.start_link([])
-
-    assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
-    assert Application.get_env(:idna, :test_key) == [live: 15, com: 35]
-    assert Application.get_env(:quack, :test_key) == [:test_value1, :test_value2]
-    assert Application.get_env(:logger, :level) == :debug
-    assert Application.get_env(:postgrex, :test_key) == :value
-
-    on_exit(fn ->
-      Application.delete_env(:pleroma, :test_key)
-      Application.delete_env(:idna, :test_key)
-      Application.delete_env(:quack, :test_key)
-      Application.delete_env(:postgrex, :test_key)
-      Application.put_env(:logger, :level, initial)
-    end)
-  end
-
-  test "transfer config values for 1 group and some keys" do
-    level = Application.get_env(:quack, :level)
-    meta = Application.get_env(:quack, :meta)
-
-    insert(:config, group: :quack, key: :level, value: :info)
-    insert(:config, group: :quack, key: :meta, value: [:none])
-
-    TransferTask.start_link([])
-
-    assert Application.get_env(:quack, :level) == :info
-    assert Application.get_env(:quack, :meta) == [:none]
-    default = Pleroma.Config.Holder.default_config(:quack, :webhook_url)
-    assert Application.get_env(:quack, :webhook_url) == default
-
-    on_exit(fn ->
-      Application.put_env(:quack, :level, level)
-      Application.put_env(:quack, :meta, meta)
-    end)
-  end
-
-  test "transfer config values with full subkey update" do
-    clear_config(:emoji)
-    clear_config(:assets)
-
-    insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
-    insert(:config, key: :assets, value: [mascots: [a: 1, b: 2]])
-
-    TransferTask.start_link([])
-
-    emoji_env = Application.get_env(:pleroma, :emoji)
-    assert emoji_env[:groups] == [a: 1, b: 2]
-    assets_env = Application.get_env(:pleroma, :assets)
-    assert assets_env[:mascots] == [a: 1, b: 2]
-  end
-
-  describe "pleroma restart" do
-    setup do
-      on_exit(fn -> Restarter.Pleroma.refresh() end)
-    end
-
-    test "don't restart if no reboot time settings were changed" do
-      clear_config(:emoji)
-      insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
-
-      refute String.contains?(
-               capture_log(fn -> TransferTask.start_link([]) end),
-               "pleroma restarted"
-             )
-    end
-
-    test "on reboot time key" do
-      clear_config(:chat)
-      insert(:config, key: :chat, value: [enabled: false])
-      assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
-    end
-
-    test "on reboot time subkey" do
-      clear_config(Pleroma.Captcha)
-      insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
-      assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
-    end
-
-    test "don't restart pleroma on reboot time key and subkey if there is false flag" do
-      clear_config(:chat)
-      clear_config(Pleroma.Captcha)
-
-      insert(:config, key: :chat, value: [enabled: false])
-      insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
-
-      refute String.contains?(
-               capture_log(fn -> TransferTask.load_and_update_env([], false) end),
-               "pleroma restarted"
-             )
-    end
-  end
-end

test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   use Pleroma.Web.ConnCase
   use Oban.Testing, repo: Pleroma.Repo
 
-  import ExUnit.CaptureLog
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
@@ -322,28 +321,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     setup do: clear_config(:configurable_from_database, true)
 
     test "pleroma restarts", %{conn: conn} do
-      capture_log(fn ->
-        assert conn |> get("/api/pleroma/admin/restart") |> json_response(200) == %{}
-      end) =~ "pleroma restarted"
-
-      refute Restarter.Pleroma.need_reboot?()
+      assert conn |> get("/api/pleroma/admin/restart") |> json_response(200) == %{}
+      refute Pleroma.Application.ConfigDependentDeps.need_reboot?()
     end
   end
 
-  test "need_reboot flag", %{conn: conn} do
-    assert conn
-           |> get("/api/pleroma/admin/need_reboot")
-           |> json_response(200) == %{"need_reboot" => false}
-
-    Restarter.Pleroma.need_reboot()
-
-    assert conn
-           |> get("/api/pleroma/admin/need_reboot")
-           |> json_response(200) == %{"need_reboot" => true}
-
-    on_exit(fn -> Restarter.Pleroma.refresh() end)
-  end
-
   describe "GET /api/pleroma/admin/users/:nickname/statuses" do
     setup do
       user = insert(:user)
@@ -999,10 +981,3 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     end
   end
 end
-
-# Needed for testing
-defmodule Pleroma.Web.Endpoint.NotReal do
-end
-
-defmodule Pleroma.Captcha.NotReal do
-end

test/pleroma/web/admin_api/controllers/config_controller_test.exs

@@ -188,7 +188,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
         Application.delete_env(:pleroma, Pleroma.Captcha.NotReal)
         Application.put_env(:pleroma, :http, http)
         Application.put_env(:tesla, :adapter, Tesla.Mock)
-        Restarter.Pleroma.refresh()
       end)
     end
 
@@ -620,7 +619,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
         value: []
       )
 
-      Pleroma.Config.TransferTask.load_and_update_env([], false)
+      Pleroma.Application.Environment.load_from_db_and_update()
 
       assert Application.get_env(:logger, :backends) == []
 
@@ -686,31 +685,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
     end
 
     test "update config setting & delete with fallback to default value", %{
-      conn: conn,
-      admin: admin,
-      token: token
+      conn: conn
     } do
       ueberauth = Application.get_env(:ueberauth, Ueberauth)
-      insert(:config, key: :keyaa1)
-      insert(:config, key: :keyaa2)
 
-      config3 =
-        insert(:config,
-          group: :ueberauth,
-          key: Ueberauth
-        )
-
-      conn =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/pleroma/admin/config", %{
-          configs: [
-            %{group: ":pleroma", key: ":keyaa1", value: "another_value"},
-            %{group: ":pleroma", key: ":keyaa2", value: "another_value"}
-          ]
-        })
-
-      assert json_response_and_validate_schema(conn, 200) == %{
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/pleroma/admin/config", %{
+               configs: [
+                 %{group: ":pleroma", key: ":keyaa1", value: "another_value"},
+                 %{group: ":pleroma", key: ":keyaa2", value: "another_value"},
+                 %{group: ":ueberauth", key: "Ueberauth", value: "another_value"}
+               ]
+             })
+             |> json_response_and_validate_schema(200) == %{
                "configs" => [
                  %{
                    "group" => ":pleroma",
@@ -723,6 +711,12 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
                    "key" => ":keyaa2",
                    "value" => "another_value",
                    "db" => [":keyaa2"]
+                 },
+                 %{
+                   "db" => ["Ueberauth"],
+                   "group" => ":ueberauth",
+                   "key" => "Ueberauth",
+                   "value" => "another_value"
                  }
                ],
                "need_reboot" => false
@@ -730,25 +724,21 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
 
       assert Application.get_env(:pleroma, :keyaa1) == "another_value"
       assert Application.get_env(:pleroma, :keyaa2) == "another_value"
-      assert Application.get_env(:ueberauth, Ueberauth) == config3.value
+      assert Application.get_env(:ueberauth, Ueberauth) == "another_value"
 
-      conn =
-        build_conn()
-        |> assign(:user, admin)
-        |> assign(:token, token)
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/pleroma/admin/config", %{
-          configs: [
-            %{group: ":pleroma", key: ":keyaa2", delete: true},
-            %{
-              group: ":ueberauth",
-              key: "Ueberauth",
-              delete: true
-            }
-          ]
-        })
-
-      assert json_response_and_validate_schema(conn, 200) == %{
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/pleroma/admin/config", %{
+               configs: [
+                 %{group: ":pleroma", key: ":keyaa2", delete: true},
+                 %{
+                   group: ":ueberauth",
+                   key: "Ueberauth",
+                   delete: true
+                 }
+               ]
+             })
+             |> json_response_and_validate_schema(200) == %{
                "configs" => [],
                "need_reboot" => false
              }
@@ -1454,3 +1444,10 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
     end
   end
 end
+
+# Needed for testing
+defmodule Pleroma.Web.Endpoint.NotReal do
+end
+
+defmodule Pleroma.Captcha.NotReal do
+end

test/support/data_case.ex

@@ -48,19 +48,11 @@ defmodule Pleroma.DataCase do
   end
 
   def clear_cachex do
-    Pleroma.Supervisor
-    |> Supervisor.which_children()
-    |> Enum.each(fn
-      {name, _, _, [Cachex]} ->
-        name
-        |> to_string
-        |> String.trim_leading("cachex_")
-        |> Kernel.<>("_cache")
-        |> String.to_existing_atom()
-        |> Cachex.clear()
-
-      _ ->
-        nil
+    Pleroma.Application.StartUpDependencies.cachex_deps()
+    |> Enum.each(fn {name, _} ->
+      "#{name}_cache"
+      |> String.to_existing_atom()
+      |> Cachex.clear()
     end)
   end