diff --git a/config/config.exs b/config/config.exs index e58454d68..203758a63 100644 --- a/config/config.exs +++ b/config/config.exs @@ -56,20 +56,6 @@ config :pleroma, Pleroma.Captcha, seconds_valid: 60, method: Pleroma.Captcha.Kocaptcha -config :pleroma, :hackney_pools, - federation: [ - max_connections: 50, - timeout: 150_000 - ], - media: [ - max_connections: 50, - timeout: 150_000 - ], - upload: [ - max_connections: 25, - timeout: 300_000 - ] - config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" # Upload configuration @@ -186,20 +172,13 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, send_user_agent: true, - adapter: [ - ssl_options: [ - # Workaround for remote server certificate chain issues - partial_chain: &:hackney_connect.partial_chain/1, - # We don't support TLS v1.3 yet - versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"] - ] - ] + adapter: [] config :pleroma, :instance, name: "Pleroma", @@ -569,6 +548,20 @@ config :pleroma, :rate_limit, config :pleroma, Pleroma.ActivityExpiration, enabled: true +config :pleroma, :gun_pools, + federation: [ + max_connections: 50, + timeout: 150_000 + ], + media: [ + max_connections: 50, + timeout: 150_000 + ], + upload: [ + max_connections: 25, + timeout: 300_000 + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 567780987..f35815e7f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -86,6 +86,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 483ac1f39..00d89f4c4 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Application do Pleroma.ActivityExpirationWorker ] ++ cachex_children() ++ - hackney_pool_children() ++ + gun_pools() ++ [ Pleroma.Web.Federator.RetryQueue, Pleroma.Stats, @@ -95,20 +95,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -157,10 +143,16 @@ defmodule Pleroma.Application do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) + defp gun_pools do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do + for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do + %{ + id: :"gun_pool_#{pool_name}", + start: {Pleroma.Gun.Connections, :start_link, [{pool_name, opts}]} + } + end + else + [] end end diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex new file mode 100644 index 000000000..43ee7f354 --- /dev/null +++ b/lib/pleroma/gun/api/api.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok + @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} + + def open(host, port, opts), do: api().open(host, port, opts) + + def info(pid), do: api().info(pid) + + def close(pid), do: api().close(pid) + + def await_up(pid), do: api().await_up(pid) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) +end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex new file mode 100644 index 000000000..603dd700e --- /dev/null +++ b/lib/pleroma/gun/api/gun.ex @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Gun do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + + @impl API + def open(host, port, opts) do + :gun.open(host, port, Map.take(opts, @gun_keys)) + end + + @impl API + def info(pid), do: :gun.info(pid) + + @impl API + def close(pid), do: :gun.close(pid) + + @impl API + def await_up(pid), do: :gun.await_up(pid) + + @impl API + def connect(pid, opts), do: :gun.connect(pid, opts) + + @impl API + def await(pid, ref), do: :gun.await(pid, ref) +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..5e1bb8abc --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API + def open(domain, 80, %{genserver_pid: genserver_pid}) + when domain in ['another-domain.com', 'some-domain.com'] do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + + send(genserver_pid, {:gun_up, conn_pid, :http}) + {:ok, conn_pid} + end + + @impl API + def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + + send(genserver_pid, {:gun_up, conn_pid, :http2}) + {:ok, conn_pid} + end + + @impl API + def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down.com', + origin_port: 80 + }) + + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + {:ok, conn_pid} + end + + @impl API + def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down_and_up.com', + origin_port: 80 + }) + + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + + {:ok, _} = + Task.start_link(fn -> + Process.sleep(500) + + send(genserver_pid, {:gun_up, conn_pid, :http}) + end) + + {:ok, conn_pid} + end + + @impl API + def open({127, 0, 0, 1}, 8123, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def open('localhost', 9050, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def await_up(_pid) do + {:ok, :http} + end + + @impl API + def connect(pid, %{host: _, port: 80}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def await(pid, ref) do + [{_, ^pid}] = Registry.lookup(API.Mock, ref) + {:response, :fin, 200, []} + end + + @impl API + def info(pid) do + [{_, info}] = Registry.lookup(API.Mock, pid) + info + end + + @impl API + def close(_pid), do: :ok +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..906607b28 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type gun_state :: :open | :up | :down + @type conn_state :: :init | :active | :idle + + @type t :: %__MODULE__{ + conn: pid(), + gun_state: gun_state(), + waiting_pids: [pid()], + conn_state: conn_state(), + used_by: [pid()], + last_reference: pos_integer(), + crf: float() + } + + defstruct conn: nil, + gun_state: :open, + waiting_pids: [], + conn_state: :init, + used_by: [], + last_reference: :os.system_time(:second), + crf: 1 +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex new file mode 100644 index 000000000..e3d392de7 --- /dev/null +++ b/lib/pleroma/gun/connections.ex @@ -0,0 +1,304 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Connections do + use GenServer + require Logger + + @type domain :: String.t() + @type conn :: Pleroma.Gun.Conn.t() + + @type t :: %__MODULE__{ + conns: %{domain() => conn()}, + opts: keyword() + } + + defstruct conns: %{}, opts: [], queue: [] + + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + + @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore + def start_link({name, opts}) do + GenServer.start_link(__MODULE__, opts, name: name) + end + + @impl true + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} + + @spec checkin(String.t(), keyword(), atom()) :: pid() + def checkin(url, opts \\ [], name \\ :default) do + opts = Enum.into(opts, %{}) + + uri = URI.parse(url) + + opts = + if uri.scheme == "https" and uri.port != 443, + do: Map.put(opts, :transport, :tls), + else: opts + + opts = + if uri.scheme == "https" do + host = uri.host |> to_charlist() + + tls_opts = + Map.get(opts, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Map.put(opts, :tls_opts, tls_opts) + else + opts + end + + GenServer.call( + name, + {:checkin, %{opts: opts, uri: uri}} + ) + end + + @spec alive?(atom()) :: boolean() + def alive?(name \\ :default) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + + @spec get_state(atom()) :: t() + def get_state(name \\ :default) do + GenServer.call(name, {:state}) + end + + def checkout(conn, pid, name \\ :default) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + def process_queue(name \\ :default) do + GenServer.cast(name, {:process_queue}) + end + + @impl true + def handle_cast({:checkout, conn_pid, pid}, state) do + {key, conn} = find_conn(state.conns, conn_pid) + used_by = List.keydelete(conn.used_by, pid, 0) + conn_state = if used_by == [], do: :idle, else: conn.conn_state + state = put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + {:noreply, state} + end + + @impl true + def handle_cast({:process_queue}, state) do + case state.queue do + [{from, key, uri, opts} | _queue] -> + try_to_checkin(key, uri, from, state, Map.put(opts, :from_cast, true)) + + [] -> + {:noreply, state} + end + end + + @impl true + def handle_call({:checkin, %{opts: opts, uri: uri}}, from, state) do + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> + time = current_time() + last_reference = time - current_conn.last_reference + + current_crf = crf(last_reference, 100, current_conn.crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] + }) + + {:reply, conn, state} + + %{gun_state: gun_state, waiting_pids: pids} when gun_state in [:open, :down] -> + state = put_in(state.conns[key].waiting_pids, [from | pids]) + {:noreply, state} + + nil -> + max_connections = state.opts[:max_connections] + + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, from, state, opts) + else + try_to_checkin(key, uri, from, state, opts) + end + end + end + + @impl true + def handle_call({:state}, _from, state), do: {:reply, state, state} + + defp try_to_checkin(key, uri, from, state, opts) do + unused_conns = + state.conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.waiting_pids == [] and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf < y.crf and x.last_reference < y.last_reference + end) + + case unused_conns do + [{close_key, least_used} | _conns] -> + :ok = API.close(least_used.conn) + + state = + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) + + open_conn(key, uri, from, state, opts) + + [] -> + queue = + if List.keymember?(state.queue, from, 0), + do: state.queue, + else: state.queue ++ [{from, key, uri, opts}] + + state = put_in(state.queue, queue) + {:noreply, state} + end + end + + @impl true + def handle_info({:gun_up, conn_pid, _protocol}, state) do + conn_key = compose_key_gun_info(conn_pid) + {key, conn} = find_conn(state.conns, conn_pid, conn_key) + + # Update state of the current connection and set waiting_pids to empty list + time = current_time() + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) + + state = + put_in(state.conns[key], %{ + conn + | gun_state: :up, + waiting_pids: [], + last_reference: time, + crf: current_crf, + conn_state: :active, + used_by: conn.waiting_pids ++ conn.used_by + }) + + # Send to all waiting processes connection pid + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) + + {:noreply, state} + end + + @impl true + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do + # we can't get info on this pid, because pid is dead + {key, conn} = find_conn(state.conns, conn_pid) + + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) + + state = put_in(state.conns[key].gun_state, :down) + {:noreply, state} + end + + defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}" + + defp compose_key_gun_info(pid) do + info = API.info(pid) + "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" + end + + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + + defp find_conn(conns, conn_pid, conn_key) do + Enum.find(conns, fn {key, conn} -> + key == conn_key and conn.conn == conn_pid + end) + end + + defp open_conn(key, uri, from, state, %{proxy: {proxy_host, proxy_port}} = opts) do + host = to_charlist(uri.host) + port = uri.port + + tls_opts = Map.get(opts, :tls_opts, []) + connect_opts = %{host: host, port: port} + + connect_opts = + if uri.scheme == "https" do + Map.put(connect_opts, :protocols, [:http2]) + |> Map.put(:transport, :tls) + |> Map.put(:tls_opts, tls_opts) + else + connect_opts + end + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + state = + put_in(state.conns[key], %Conn{ + conn: conn, + waiting_pids: [], + gun_state: :up, + conn_state: :active, + used_by: [from] + }) + + if opts[:from_cast] do + GenServer.reply(from, conn) + end + + {:reply, conn, state} + else + error -> + Logger.warn(inspect(error)) + {:reply, nil, state} + end + end + + defp open_conn(key, uri, from, state, opts) do + host = to_charlist(uri.host) + port = uri.port + + with {:ok, conn} <- API.open(host, port, opts) do + state = + if opts[:from_cast] do + put_in(state.queue, List.keydelete(state.queue, from, 0)) + else + state + end + + state = + put_in(state.conns[key], %Conn{ + conn: conn, + waiting_pids: [from] + }) + + {:noreply, state} + else + error -> + Logger.warn(inspect(error)) + {:reply, nil, state} + end + end + + defp current_time do + :os.system_time(:second) + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..d4e6d0f99 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -7,14 +7,13 @@ defmodule Pleroma.HTTP.Connection do Connection for http-requests. """ - @hackney_options [ + @options [ connect_timeout: 10_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, + timeout: 20_000, pool: :federation ] - @adapter Application.get_env(:tesla, :adapter) + + require Logger @doc """ Configure a client connection @@ -25,19 +24,108 @@ defmodule Pleroma.HTTP.Connection do """ @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + middleware = [Tesla.Middleware.FollowRedirects] + adapter = Application.get_env(:tesla, :adapter) + Tesla.client(middleware, {adapter, options(opts)}) end - # fetch Hackney options + # fetch http options # - def hackney_options(opts) do + def options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) + proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @hackney_options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + proxy = + case parse_proxy(proxy_url) do + {:ok, proxy_host, proxy_port} -> {proxy_host, proxy_port} + _ -> nil + end + + options = + @options + |> Keyword.merge(adapter_options) + |> Keyword.merge(options) + |> Keyword.merge(proxy: proxy) + + pool = options[:pool] + url = options[:url] + + if not is_nil(url) and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do + get_conn_for_gun(url, options, pool) + else + options + end + end + + defp get_conn_for_gun(url, options, pool) do + case Pleroma.Gun.Connections.checkin(url, options, pool) do + nil -> + options + + conn -> + %{host: host, port: port} = URI.parse(url) + + # verify sertificates opts for gun + tls_opts = [ + verify: :verify_peer, + cacerts: :certifi.cacerts(), + depth: 20, + server_name_indication: to_charlist(host), + reuse_sessions: false, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: to_charlist(host)]} + ] + + Keyword.put(options, :conn, conn) + |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") + |> Keyword.put(:tls_opts, tls_opts) + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {tuple, pos_integer()} | {:error, atom()} | nil + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + :error -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {_type, host, port} <- proxy do + {:ok, parse_host(host), port} + else + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + @spec parse_host(String.t() | tuple()) :: charlist() | atom() + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..0a7db737f 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -28,21 +28,44 @@ defmodule Pleroma.HTTP do """ def request(method, url, body \\ "", headers \\ [], options \\ []) do try do + options = process_request_options(options) + + adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + options = - process_request_options(options) - |> process_sni_options(url) + if adapter_gun? do + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:url, url) + + Keyword.put(options, :adapter, adapter_opts) + else + options + end params = Keyword.get(options, :params, []) - %{} - |> Builder.method(method) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.url(url) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() + request = + %{} + |> Builder.method(method) + |> Builder.url(url) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Enum.into([]) + + client = Connection.new(options) + response = Tesla.request(client, request) + + if adapter_gun? do + %{adapter: {_, _, [adapter_options]}} = client + pool = adapter_options[:pool] + Pleroma.Gun.Connections.checkout(adapter_options[:conn], self(), pool) + Pleroma.Gun.Connections.process_queue(pool) + end + + response rescue e -> {:error, e} @@ -52,20 +75,8 @@ defmodule Pleroma.HTTP do end end - defp process_sni_options(options, nil), do: options - - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() - - case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options - end - end - def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + Keyword.merge(Pleroma.HTTP.Connection.options([]), options) end @doc """ diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, header_list) do header_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list ++ [{"user-agent", Pleroma.Application.user_agent()}] else header_list end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c1795ae0f..1dfba04eb 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -108,7 +108,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -120,7 +120,7 @@ defmodule Pleroma.Object.Fetcher do date = Pleroma.Signature.signed_date() headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..71c2b2911 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,9 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} @@ -14,8 +19,8 @@ defmodule Pleroma.ReverseProxy.Client do @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) end end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..fad577ec1 --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-onl + +defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + + @adapters [Tesla.Adapter.Gun] + + def request(method, url, headers, body, opts \\ []) do + adapter_opts = + Keyword.get(opts, :adapter, []) + |> Keyword.put(:body_as, :chunks) + + with {:ok, response} <- + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, adapter_opts) + ) do + if is_map(response.body), + do: {:ok, response.status, response.headers, response.body}, + else: {:ok, response.status, response.headers} + else + {:error, error} -> {:error, error} + end + end + + def stream_body(%{fin: true}), do: :done + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} + {:nofin, part} -> {:ok, part, client} + {:error, error} -> {:error, error} + end + end + + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter.read_chunk(pid, stream, opts) + end + + def close(pid) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support closing connection" + end + + adapter.close(pid) + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 03efad30a..df4eca207 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -58,10 +58,10 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [gun](https://github.com/ninenines/gun). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -93,9 +93,9 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) - |> Keyword.merge(@default_hackney_options) + client_opts = + Pleroma.HTTP.Connection.options([]) + |> Keyword.merge(@default_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() @@ -108,7 +108,7 @@ defmodule Pleroma.ReverseProxy do opts end - with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + with {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -147,11 +147,11 @@ defmodule Pleroma.ReverseProxy do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -201,7 +201,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index a179dd54d..52ef0167c 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ + @options [ pool: :media, recv_timeout: 10_000 ] @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index a10cc779b..25e91eee6 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -95,7 +95,6 @@ defmodule Pleroma.Web.AdminAPI.Config do end defp do_convert({:dispatch, [entity]}), do: %{"tuple" => [":dispatch", [inspect(entity)]]} - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} defp do_convert(entity) when is_tuple(entity), do: %{"tuple" => do_convert(Tuple.to_list(entity))} @@ -129,11 +128,6 @@ defmodule Pleroma.Web.AdminAPI.Config do {:dispatch, [dispatch_settings]} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = do_eval(entity) - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 331cbc0b7..1c400d9e4 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -373,7 +373,7 @@ defmodule Pleroma.Web.OStatus do {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( url, - [{:Accept, "application/atom+xml"}] + [{"accept", "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body, options) diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index d376e2069..947234aa2 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) data = Floki.attribute(html, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358c..ade4ac891 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -78,7 +78,7 @@ defmodule Pleroma.Web.RichMedia.Parser do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) html |> maybe_parse() diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ecb39ee50..624ee5ef7 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -217,7 +217,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) diff --git a/mix.exs b/mix.exs index 3170d6f2d..a7ed3291e 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,15 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.2"}, + { + :tesla, + github: "alex-strizhakov/tesla", + ref: "199e77f6e4390495eef7c31f2d830da855571b64", + override: true + }, + {:cowlib, "~> 2.7.3", override: true}, + {:gun, + github: "ninenines/gun", ref: "491ddf58c0e14824a741852fdc522b390b306ae2", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 2639e96e9..3a6ef325c 100644 --- a/mix.lock +++ b/mix.lock @@ -37,6 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, + "gun": {:git, "https://github.com/ninenines/gun.git", "491ddf58c0e14824a741852fdc522b390b306ae2", [ref: "491ddf58c0e14824a741852fdc522b390b306ae2"]}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -84,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "199e77f6e4390495eef7c31f2d830da855571b64", [ref: "199e77f6e4390495eef7c31f2d830da855571b64"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs new file mode 100644 index 000000000..39f77070a --- /dev/null +++ b/test/gun/connections_test.exs @@ -0,0 +1,685 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Gun.ConnectionsTest do + use ExUnit.Case + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + alias Pleroma.Gun.Connections + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) + :ok + end + + setup do + name = :test_gun_connections + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + {:ok, pid} = Connections.start_link({name, [max_connections: 2, timeout: 10]}) + + {:ok, name: name, pid: pid} + end + + describe "alive?/2" do + test "is alive", %{name: name} do + assert Connections.alive?(name) + end + + test "returns false if not started" do + refute Connections.alive?(:some_random_name) + end + end + + test "opens connection and reuse it on next request", %{name: name, pid: pid} do + conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + self = self() + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}, {^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [], + conn_state: :idle + } + } + } = Connections.get_state(name) + end + + test "reuses connection based on protocol", %{name: name, pid: pid} do + conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + assert is_pid(conn) + assert Process.alive?(conn) + + https_conn = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) + + refute conn == https_conn + + reused_https = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) + + refute conn == reused_https + + assert reused_https == https_conn + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + }, + "https:some-domain.com:443" => %Conn{ + conn: ^https_conn, + gun_state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "process gun_down message", %{name: name, pid: pid} do + conn = Connections.checkin("http://gun_down.com", [genserver_pid: pid], name) + + refute conn + + %Connections{ + conns: %{ + "http:gun_down.com:80" => %Conn{ + conn: _, + gun_state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + end + + test "process gun_down message and then gun_up", %{name: name, pid: pid} do + conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name) + + refute conn + + %Connections{ + conns: %{ + "http:gun_down_and_up.com:80" => %Conn{ + conn: _, + gun_state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + + conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + %Connections{ + conns: %{ + "http:gun_down_and_up.com:80" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "async processes get same conn for same domain", %{name: name, pid: pid} do + tasks = + for _ <- 1..5 do + Task.async(fn -> + Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + end) + end + + tasks_with_results = Task.yield_many(tasks) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + res || Task.shutdown(task, :brutal_kill) + end) + + conns = for {:ok, value} <- results, do: value + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: conn, + gun_state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + + assert Enum.all?(conns, fn res -> res == conn end) + end + + test "remove frequently used and idle", %{name: name, pid: pid} do + self = self() + conn1 = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) + + [conn2 | _conns] = + for _ <- 1..4 do + Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}] + }, + "https:some-domain.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.checkout(conn1, self, name) + + conn = Connections.checkin("http://another-domain.com", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:another-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + }, + "http:some-domain.com:80" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + + describe "integration test" do + @describetag :integration + + test "opens connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.checkin("http://httpbin.org", [], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.checkin("http://httpbin.org", [], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "opens ssl connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.checkin("https://httpbin.org", [], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.checkin("https://httpbin.org", [], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "remove frequently used and idle", %{name: name, pid: pid} do + self = self() + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + + for _ <- 1..4 do + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + }, + "https:www.google.com:443" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + + test "remove earlier used and idle", %{name: name, pid: pid} do + self = self() + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + + Process.sleep(1_000) + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + :ok = Connections.checkout(conn, self, name) + Process.sleep(1_000) + conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + + test "doesn't drop active connections on pool overflow addinng new requests to the queue", %{ + name: name, + pid: pid + } do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + self = self() + Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn1 = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn2 = Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}, {^self, _}] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + task = + Task.async(fn -> Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) end) + + task_pid = task.pid + + :ok = Connections.checkout(conn1, self, name) + + Process.sleep(1_000) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + } + }, + queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.checkout(conn1, self, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :idle, + used_by: [] + } + }, + queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.process_queue(name) + conn = Task.await(task) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^task_pid, _}] + } + }, + queue: [], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + end + + describe "with proxy usage" do + test "proxy as ip", %{name: name, pid: pid} do + conn = + Connections.checkin( + "http://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + %Connections{ + conns: %{ + "http:proxy_string.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.checkin( + "http://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + assert reused_conn == conn + end + + test "proxy as host", %{name: name, pid: pid} do + conn = + Connections.checkin( + "http://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + %Connections{ + conns: %{ + "http:proxy_tuple_atom.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.checkin( + "http://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + assert reused_conn == conn + end + + test "proxy as ip and ssl", %{name: name, pid: pid} do + conn = + Connections.checkin( + "https://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + %Connections{ + conns: %{ + "https:proxy_string.com:443" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.checkin( + "https://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + assert reused_conn == conn + end + + test "proxy as host and ssl", %{name: name, pid: pid} do + conn = + Connections.checkin( + "https://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + %Connections{ + conns: %{ + "https:proxy_tuple_atom.com:443" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.checkin( + "https://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + assert reused_conn == conn + end + end + + describe "crf/3" do + setup do + crf = Connections.crf(1, 10, 1) + {:ok, crf: crf} + end + + test "more used will have crf higher", %{crf: crf} do + # used 3 times + crf1 = Connections.crf(1, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 > crf2 + end + + test "recently used will have crf higher on equal references", %{crf: crf} do + # used 4 sec ago + crf1 = Connections.crf(3, 10, crf) + + # used 3 sec ago + crf2 = Connections.crf(4, 10, crf) + + assert crf1 > crf2 + end + + test "equal crf on equal reference and time", %{crf: crf} do + # used 2 times + crf1 = Connections.crf(1, 10, crf) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 == crf2 + end + + test "recently used will have higher crf", %{crf: crf} do + crf1 = Connections.crf(2, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + crf2 = Connections.crf(3, 10, crf) + crf2 = Connections.crf(4, 10, crf2) + assert crf1 > crf2 + end + end +end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs new file mode 100644 index 000000000..99eab4026 --- /dev/null +++ b/test/http/connection_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.HTTP.ConnectionTest do + use ExUnit.Case, async: true + import ExUnit.CaptureLog + alias Pleroma.HTTP.Connection + + describe "parse_host/1" do + test "as atom" do + assert Connection.parse_host(:localhost) == 'localhost' + end + + test "as string" do + assert Connection.parse_host("localhost.com") == 'localhost.com' + end + + test "as string ip" do + assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1} + end + end + + describe "parse_proxy/1" do + test "ip with port" do + assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123} + end + + test "host with port" do + assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123} + end + + test "as tuple" do + assert Connection.parse_proxy({:socks5, :localhost, 9050}) == {:ok, 'localhost', 9050} + end + + test "as tuple with string host" do + assert Connection.parse_proxy({:socks5, "localhost", 9050}) == {:ok, 'localhost', 9050} + end + + test "ip without port" do + capture_log(fn -> + assert Connection.parse_proxy("127.0.0.1") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"127.0.0.1\"" + end + + test "host without port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"localhost\"" + end + + test "host with bad port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost:port") == {:error, :error_parsing_port_in_proxy} + end) =~ "parsing port in proxy fail \"localhost:port\"" + end + + test "as tuple without port" do + capture_log(fn -> + assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail {:socks5, :localhost}" + end + + test "with nil" do + assert Connection.parse_proxy(nil) == nil + end + end +end diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 170ca916f..77a1e870a 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.RequestBuilderTest do - use ExUnit.Case, async: true + use ExUnit.Case use Pleroma.Tests.Helpers alias Pleroma.HTTP.RequestBuilder @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do Pleroma.Config.put([:http, :send_user_agent], true) assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", Pleroma.Application.user_agent()}] + headers: [{"user-agent", Pleroma.Application.user_agent()}] } end end diff --git a/test/http_test.exs b/test/http_test.exs index 5f9522cf0..b88e3b605 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -56,4 +56,33 @@ defmodule Pleroma.HTTPTest do } end end + + @tag :integration + test "get_conn_for_gun/3" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + + options = [adapter: [pool: :federation]] + + assert {:ok, resp} = Pleroma.HTTP.get("https://httpbin.org/user-agent", [], options) + + adapter_opts = resp.opts[:adapter] + + assert resp.status == 200 + + assert adapter_opts[:url] == "https://httpbin.org/user-agent" + state = Pleroma.Gun.Connections.get_state(:federation) + conn = state.conns["https:httpbin.org:443"] + + assert conn.conn_state == :idle + assert conn.used_by == [] + assert state.queue == [] + end end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs new file mode 100644 index 000000000..c9ea3f6c0 --- /dev/null +++ b/test/reverse_proxy/client/tesla_test.exs @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxy.Client.TeslaTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla + + setup_all do + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Mock) + end) + end + + defp check_ref(%{pid: pid, stream: stream} = ref) do + assert is_pid(pid) + assert is_reference(stream) + assert ref[:fin] + end + + defp close(%{pid: pid}) do + Pleroma.ReverseProxy.Client.Tesla.close(pid) + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs similarity index 67% rename from test/reverse_proxy_test.exs rename to test/reverse_proxy/reverse_proxy_test.exs index 3a83c4c48..b2f7932bf 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import ExUnit.CaptureLog import Mox alias Pleroma.ReverseProxy @@ -29,11 +29,11 @@ defmodule Pleroma.ReverseProxyTest do {"content-length", byte_size(json) |> to_string()} ], %{url: url}} end) - |> expect(:stream_body, invokes, fn %{url: url} -> + |> expect(:stream_body, invokes, fn %{url: url} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json} + {:ok, json, client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -66,6 +66,38 @@ defmodule Pleroma.ReverseProxyTest do assert conn.halted end + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + describe "max_body " do test "length returns error if content-length more than option", %{conn: conn} do user_agent_mock("hackney/1.15.1", 0) @@ -76,38 +108,6 @@ defmodule Pleroma.ReverseProxyTest do "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" end - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) @@ -179,12 +179,12 @@ defmodule Pleroma.ReverseProxyTest do Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers})} + {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -261,11 +261,11 @@ defmodule Pleroma.ReverseProxyTest do {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, ""} + {:ok, "", client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") @@ -297,4 +297,73 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end + + describe "integration tests" do + @describetag :integration + + test "with tesla client with gun adapter", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end + + test "with tesla client with gun adapter with ssl", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end + + test "tesla client with gun client follow redirects", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") + + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 55b141dd8..2ac25f6ec 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -91,7 +91,7 @@ defmodule HttpRequestMock do "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -104,7 +104,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -125,7 +125,7 @@ defmodule HttpRequestMock do "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -151,7 +151,7 @@ defmodule HttpRequestMock do "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -172,7 +172,7 @@ defmodule HttpRequestMock do "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -185,7 +185,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -198,7 +198,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd/?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -211,7 +211,7 @@ defmodule HttpRequestMock do "https://mst3k.interlinked.me/users/luciferMysticus", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -232,7 +232,7 @@ defmodule HttpRequestMock do "https://hubzilla.example.org/channel/kaniini", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -241,7 +241,7 @@ defmodule HttpRequestMock do }} end - def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -249,7 +249,7 @@ defmodule HttpRequestMock do }} end - def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -268,7 +268,7 @@ defmodule HttpRequestMock do }} end - def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do + def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -324,7 +324,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/admin", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -332,7 +334,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/gargron", _, _, [ + {"accept", "application/activity+json"} + ]) do {:error, :nxdomain} end @@ -340,7 +344,7 @@ defmodule HttpRequestMock do "http://mastodon.example.org/@admin/99541947525187367", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -357,7 +361,7 @@ defmodule HttpRequestMock do }} end - def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do + def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -369,7 +373,7 @@ defmodule HttpRequestMock do "https://mstdn.io/users/mayuutann/statuses/99568293732299394", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -389,7 +393,7 @@ defmodule HttpRequestMock do }} end - def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") + def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" @@ -416,7 +420,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -460,7 +464,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -513,7 +517,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -526,7 +530,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", _, _, - Accept: "application/atom+xml" + [{"accept", "application/atom+xml"}] ) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end @@ -543,7 +547,7 @@ defmodule HttpRequestMock do "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -565,7 +569,7 @@ defmodule HttpRequestMock do "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -579,7 +583,7 @@ defmodule HttpRequestMock do "http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{status: 406, body: ""}} end @@ -615,7 +619,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -628,7 +632,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -641,7 +645,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{status: 200, body: ""}} end @@ -658,7 +662,7 @@ defmodule HttpRequestMock do "http://framatube.org/main/xrd?uri=framasoft@framatube.org", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -680,7 +684,7 @@ defmodule HttpRequestMock do "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -717,7 +721,7 @@ defmodule HttpRequestMock do "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -775,7 +779,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} end - def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do + def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end @@ -892,7 +896,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -905,7 +909,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -927,7 +931,9 @@ defmodule HttpRequestMock do }} end - def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -939,7 +945,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity2.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -951,7 +959,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity3.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex new file mode 100644 index 000000000..36df1ed95 --- /dev/null +++ b/test/support/reverse_proxy_client_case.ex @@ -0,0 +1,80 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReverseProxyClientCase do + defmacro __using__(client: client) do + quote do + use ExUnit.Case + @moduletag :integration + @client unquote(client) + + setup do + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, Tesla.Mock) end) + end + + test "get response body stream" do + {:ok, status, headers, ref} = + @client.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "", + [] + ) + + assert status == 200 + assert headers != [] + + {:ok, response, ref} = @client.stream_body(ref) + check_ref(ref) + assert is_binary(response) + assert byte_size(response) == 10 + + assert :done == @client.stream_body(ref) + end + + test "head response" do + {:ok, status, headers} = @client.request(:head, "http://httpbin.org/get", [], "", []) + + assert status == 200 + assert headers != [] + end + + test "get error response" do + case @client.request( + :get, + "http://httpbin.org/status/500", + [], + "", + [] + ) do + {:ok, status, headers, ref} -> + assert status == 500 + assert headers != [] + check_ref(ref) + + assert :ok == close(ref) + + {:ok, status, headers} -> + assert headers != [] + end + end + + test "head error response" do + {:ok, status, headers} = + @client.request( + :head, + "http://httpbin.org/status/500", + [], + "", + [] + ) + + assert status == 500 + assert headers != [] + end + end + end +end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 1afdb6a50..8f43cd483 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1779,8 +1779,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":key1", nil]} ] } ] @@ -1796,8 +1795,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":key1", nil]} ] } ] diff --git a/test/web/admin_api/config_test.exs b/test/web/admin_api/config_test.exs index 3190dc1c8..d41666ef3 100644 --- a/test/web/admin_api/config_test.exs +++ b/test/web/admin_api/config_test.exs @@ -238,14 +238,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigTest do assert Config.from_binary(binary) == [key: "value"] end - test "keyword with partial_chain key" do - binary = - Config.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}]) - - assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1) - assert Config.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1] - end - test "keyword" do binary = Config.transform([ diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index fe4ffdb59..1061403f4 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Web.CommonAPI import Pleroma.Factory import Mock + import ExUnit.CaptureLog setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -367,15 +368,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert html_response(response, 200) =~ "Remote follow" end - test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + test "show follow page with error when user cannot fetch by `acct` link", %{conn: conn} do user = insert(:user) - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - assert html_response(response, 200) =~ "Error fetching user" + assert html_response(response, 200) =~ "Error fetching user" + end) =~ + "Could not decode user at fetch https://mastodon.social/users/not_found, {:error, \"Object has been deleted\"}" end end diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index e23086b2a..f940f95b6 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do import Pleroma.Factory import Tesla.Mock + import ExUnit.CaptureLog setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -75,11 +76,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do test "Sends a 404 when invalid format" do user = insert(:user) - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end + assert capture_log(fn -> + assert_raise Phoenix.NotAcceptableError, fn -> + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + end + end) =~ "Internal server error:" end test "Sends a 400 when resource param is missing" do