squeegily/pleroma
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'gun' of git.pleroma.social:alex.s/pleroma into integration/alex.s/gun

Browse Source
This commit is contained in:
Ariadne Conill 2019-08-25 21:00:58 +00:00
commit 361940e119
35 changed files with 1848 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
config/config.exs
View File

@ -56,20 +56,6 @@ config :pleroma, Pleroma.Captcha,
seconds_valid: 60,
method: Pleroma.Captcha.Kocaptcha
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
# Upload configuration
@ -186,20 +172,13 @@ config :mime, :types, %{
"application/ld+json" => ["activity+json"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
config :tesla, adapter: Tesla.Adapter.Gun
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
proxy_url: nil,
send_user_agent: true,
adapter: [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
]
]
adapter: []
config :pleroma, :instance,
name: "Pleroma",
@ -569,6 +548,20 @@ config :pleroma, :rate_limit,
config :pleroma, Pleroma.ActivityExpiration, enabled: true
config :pleroma, :gun_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

2
config/test.exs
View File

@ -86,6 +86,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3
config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock
if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs"
else

30
lib/pleroma/application.ex
View File

@ -39,7 +39,7 @@ defmodule Pleroma.Application do
Pleroma.ActivityExpirationWorker
] ++
cachex_children() ++
hackney_pool_children() ++
gun_pools() ++
[
Pleroma.Web.Federator.RetryQueue,
Pleroma.Stats,
@ -95,20 +95,6 @@ defmodule Pleroma.Application do
Pleroma.Web.Endpoint.Instrumenter.setup()
end
def enabled_hackney_pools do
[:media] ++
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
[:federation]
else
[]
end ++
if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload]
else
[]
end
end
defp cachex_children do
[
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@ -157,10 +143,16 @@ defmodule Pleroma.Application do
defp chat_child(_, _), do: []
defp hackney_pool_children do
for pool <- enabled_hackney_pools() do
options = Pleroma.Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
defp gun_pools do
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do
for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do
%{
id: :"gun_pool_#{pool_name}",
start: {Pleroma.Gun.Connections, :start_link, [{pool_name, opts}]}
}
end
else
[]
end
end

26
lib/pleroma/gun/api/api.ex Normal file
View File

@ -0,0 +1,26 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback await_up(pid) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
def open(host, port, opts), do: api().open(host, port, opts)
def info(pid), do: api().info(pid)
def close(pid), do: api().close(pid)
def await_up(pid), do: api().await_up(pid)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
end

43
lib/pleroma/gun/api/gun.ex Normal file
View File

@ -0,0 +1,43 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API.Gun do
@behaviour Pleroma.Gun.API
alias Pleroma.Gun.API
@gun_keys [
:connect_timeout,
:http_opts,
:http2_opts,
:protocols,
:retry,
:retry_timeout,
:trace,
:transport,
:tls_opts,
:tcp_opts,
:ws_opts
]
@impl API
def open(host, port, opts) do
:gun.open(host, port, Map.take(opts, @gun_keys))
end
@impl API
def info(pid), do: :gun.info(pid)
@impl API
def close(pid), do: :gun.close(pid)
@impl API
def await_up(pid), do: :gun.await_up(pid)
@impl API
def connect(pid, opts), do: :gun.connect(pid, opts)
@impl API
def await(pid, ref), do: :gun.await(pid, ref)
end

118
lib/pleroma/gun/api/mock.ex Normal file
View File

@ -0,0 +1,118 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API.Mock do
@behaviour Pleroma.Gun.API
alias Pleroma.Gun.API
@impl API
def open(domain, 80, %{genserver_pid: genserver_pid})
when domain in ['another-domain.com', 'some-domain.com'] do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "http",
origin_host: domain,
origin_port: 80
})
send(genserver_pid, {:gun_up, conn_pid, :http})
{:ok, conn_pid}
end
@impl API
def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "https",
origin_host: 'some-domain.com',
origin_port: 443
})
send(genserver_pid, {:gun_up, conn_pid, :http2})
{:ok, conn_pid}
end
@impl API
def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "http",
origin_host: 'gun_down.com',
origin_port: 80
})
send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil})
{:ok, conn_pid}
end
@impl API
def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do
{:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end)
Registry.register(API.Mock, conn_pid, %{
origin_scheme: "http",
origin_host: 'gun_down_and_up.com',
origin_port: 80
})
send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil})
{:ok, _} =
Task.start_link(fn ->
Process.sleep(500)
send(genserver_pid, {:gun_up, conn_pid, :http})
end)
{:ok, conn_pid}
end
@impl API
def open({127, 0, 0, 1}, 8123, _) do
Task.start_link(fn -> Process.sleep(1_000) end)
end
@impl API
def open('localhost', 9050, _) do
Task.start_link(fn -> Process.sleep(1_000) end)
end
@impl API
def await_up(_pid) do
{:ok, :http}
end
@impl API
def connect(pid, %{host: _, port: 80}) do
ref = make_ref()
Registry.register(API.Mock, ref, pid)
ref
end
@impl API
def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do
ref = make_ref()
Registry.register(API.Mock, ref, pid)
ref
end
@impl API
def await(pid, ref) do
[{_, ^pid}] = Registry.lookup(API.Mock, ref)
{:response, :fin, 200, []}
end
@impl API
def info(pid) do
[{_, info}] = Registry.lookup(API.Mock, pid)
info
end
@impl API
def close(_pid), do: :ok
end

29
lib/pleroma/gun/conn.ex Normal file
View File

@ -0,0 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Conn do
@moduledoc """
Struct for gun connection data
"""
@type gun_state :: :open | :up | :down
@type conn_state :: :init | :active | :idle
@type t :: %__MODULE__{
conn: pid(),
gun_state: gun_state(),
waiting_pids: [pid()],
conn_state: conn_state(),
used_by: [pid()],
last_reference: pos_integer(),
crf: float()
}
defstruct conn: nil,
gun_state: :open,
waiting_pids: [],
conn_state: :init,
used_by: [],
last_reference: :os.system_time(:second),
crf: 1
end

304
lib/pleroma/gun/connections.ex Normal file
View File

@ -0,0 +1,304 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Connections do
use GenServer
require Logger
@type domain :: String.t()
@type conn :: Pleroma.Gun.Conn.t()
@type t :: %__MODULE__{
conns: %{domain() => conn()},
opts: keyword()
}
defstruct conns: %{}, opts: [], queue: []
alias Pleroma.Gun.API
alias Pleroma.Gun.Conn
@spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore
def start_link({name, opts}) do
GenServer.start_link(__MODULE__, opts, name: name)
end
@impl true
def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
@spec checkin(String.t(), keyword(), atom()) :: pid()
def checkin(url, opts \\ [], name \\ :default) do
opts = Enum.into(opts, %{})
uri = URI.parse(url)
opts =
if uri.scheme == "https" and uri.port != 443,
do: Map.put(opts, :transport, :tls),
else: opts
opts =
if uri.scheme == "https" do
host = uri.host |> to_charlist()
tls_opts =
Map.get(opts, :tls_opts, [])
|> Keyword.put(:server_name_indication, host)
Map.put(opts, :tls_opts, tls_opts)
else
opts
end
GenServer.call(
name,
{:checkin, %{opts: opts, uri: uri}}
)
end
@spec alive?(atom()) :: boolean()
def alive?(name \\ :default) do
pid = Process.whereis(name)
if pid, do: Process.alive?(pid), else: false
end
@spec get_state(atom()) :: t()
def get_state(name \\ :default) do
GenServer.call(name, {:state})
end
def checkout(conn, pid, name \\ :default) do
GenServer.cast(name, {:checkout, conn, pid})
end
def process_queue(name \\ :default) do
GenServer.cast(name, {:process_queue})
end
@impl true
def handle_cast({:checkout, conn_pid, pid}, state) do
{key, conn} = find_conn(state.conns, conn_pid)
used_by = List.keydelete(conn.used_by, pid, 0)
conn_state = if used_by == [], do: :idle, else: conn.conn_state
state = put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
{:noreply, state}
end
@impl true
def handle_cast({:process_queue}, state) do
case state.queue do
[{from, key, uri, opts} | _queue] ->
try_to_checkin(key, uri, from, state, Map.put(opts, :from_cast, true))
[] ->
{:noreply, state}
end
end
@impl true
def handle_call({:checkin, %{opts: opts, uri: uri}}, from, state) do
key = compose_key(uri)
case state.conns[key] do
%{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up ->
time = current_time()
last_reference = time - current_conn.last_reference
current_crf = crf(last_reference, 100, current_conn.crf)
state =
put_in(state.conns[key], %{
current_conn
| last_reference: time,
crf: current_crf,
conn_state: :active,
used_by: [from | current_conn.used_by]
})
{:reply, conn, state}
%{gun_state: gun_state, waiting_pids: pids} when gun_state in [:open, :down] ->
state = put_in(state.conns[key].waiting_pids, [from | pids])
{:noreply, state}
nil ->
max_connections = state.opts[:max_connections]
if Enum.count(state.conns) < max_connections do
open_conn(key, uri, from, state, opts)
else
try_to_checkin(key, uri, from, state, opts)
end
end
end
@impl true
def handle_call({:state}, _from, state), do: {:reply, state, state}
defp try_to_checkin(key, uri, from, state, opts) do
unused_conns =
state.conns
|> Enum.filter(fn {_k, v} ->
v.conn_state == :idle and v.waiting_pids == [] and v.used_by == []
end)
|> Enum.sort(fn {_x_k, x}, {_y_k, y} ->
x.crf < y.crf and x.last_reference < y.last_reference
end)
case unused_conns do
[{close_key, least_used} | _conns] ->
:ok = API.close(least_used.conn)
state =
put_in(
state.conns,
Map.delete(state.conns, close_key)
)
open_conn(key, uri, from, state, opts)
[] ->
queue =
if List.keymember?(state.queue, from, 0),
do: state.queue,
else: state.queue ++ [{from, key, uri, opts}]
state = put_in(state.queue, queue)
{:noreply, state}
end
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do
conn_key = compose_key_gun_info(conn_pid)
{key, conn} = find_conn(state.conns, conn_pid, conn_key)
# Update state of the current connection and set waiting_pids to empty list
time = current_time()
last_reference = time - conn.last_reference
current_crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| gun_state: :up,
waiting_pids: [],
last_reference: time,
crf: current_crf,
conn_state: :active,
used_by: conn.waiting_pids ++ conn.used_by
})
# Send to all waiting processes connection pid
Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end)
{:noreply, state}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do
# we can't get info on this pid, because pid is dead
{key, conn} = find_conn(state.conns, conn_pid)
Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end)
state = put_in(state.conns[key].gun_state, :down)
{:noreply, state}
end
defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}"
defp compose_key_gun_info(pid) do
info = API.info(pid)
"#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}"
end
defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid
end)
end
defp find_conn(conns, conn_pid, conn_key) do
Enum.find(conns, fn {key, conn} ->
key == conn_key and conn.conn == conn_pid
end)
end
defp open_conn(key, uri, from, state, %{proxy: {proxy_host, proxy_port}} = opts) do
host = to_charlist(uri.host)
port = uri.port
tls_opts = Map.get(opts, :tls_opts, [])
connect_opts = %{host: host, port: port}
connect_opts =
if uri.scheme == "https" do
Map.put(connect_opts, :protocols, [:http2])
|> Map.put(:transport, :tls)
|> Map.put(:tls_opts, tls_opts)
else
connect_opts
end
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- API.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- API.await_up(conn),
stream <- API.connect(conn, connect_opts),
{:response, :fin, 200, _} <- API.await(conn, stream) do
state =
put_in(state.conns[key], %Conn{
conn: conn,
waiting_pids: [],
gun_state: :up,
conn_state: :active,
used_by: [from]
})
if opts[:from_cast] do
GenServer.reply(from, conn)
end
{:reply, conn, state}
else
error ->
Logger.warn(inspect(error))
{:reply, nil, state}
end
end
defp open_conn(key, uri, from, state, opts) do
host = to_charlist(uri.host)
port = uri.port
with {:ok, conn} <- API.open(host, port, opts) do
state =
if opts[:from_cast] do
put_in(state.queue, List.keydelete(state.queue, from, 0))
else
state
end
state =
put_in(state.conns[key], %Conn{
conn: conn,
waiting_pids: [from]
})
{:noreply, state}
else
error ->
Logger.warn(inspect(error))
{:reply, nil, state}
end
end
defp current_time do
:os.system_time(:second)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end

112
lib/pleroma/http/connection.ex
View File

@ -7,14 +7,13 @@ defmodule Pleroma.HTTP.Connection do
Connection for http-requests.
"""
@hackney_options [
@options [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
force_redirect: true,
timeout: 20_000,
pool: :federation
]
@adapter Application.get_env(:tesla, :adapter)
require Logger
@doc """
Configure a client connection
@ -25,19 +24,108 @@ defmodule Pleroma.HTTP.Connection do
"""
@spec new(Keyword.t()) :: Tesla.Env.client()
def new(opts \\ []) do
Tesla.client([], {@adapter, hackney_options(opts)})
middleware = [Tesla.Middleware.FollowRedirects]
adapter = Application.get_env(:tesla, :adapter)
Tesla.client(middleware, {adapter, options(opts)})
end
# fetch Hackney options
# fetch http options
#
def hackney_options(opts) do
def options(opts) do
options = Keyword.get(opts, :adapter, [])
adapter_options = Pleroma.Config.get([:http, :adapter], [])
proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
@hackney_options
|> Keyword.merge(adapter_options)
|> Keyword.merge(options)
|> Keyword.merge(proxy: proxy_url)
proxy =
case parse_proxy(proxy_url) do
{:ok, proxy_host, proxy_port} -> {proxy_host, proxy_port}
_ -> nil
end
options =
@options
|> Keyword.merge(adapter_options)
|> Keyword.merge(options)
|> Keyword.merge(proxy: proxy)
pool = options[:pool]
url = options[:url]
if not is_nil(url) and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do
get_conn_for_gun(url, options, pool)
else
options
end
end
defp get_conn_for_gun(url, options, pool) do
case Pleroma.Gun.Connections.checkin(url, options, pool) do
nil ->
options
conn ->
%{host: host, port: port} = URI.parse(url)
# verify sertificates opts for gun
tls_opts = [
verify: :verify_peer,
cacerts: :certifi.cacerts(),
depth: 20,
server_name_indication: to_charlist(host),
reuse_sessions: false,
verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: to_charlist(host)]}
]
Keyword.put(options, :conn, conn)
|> Keyword.put(:close_conn, false)
|> Keyword.put(:original, "#{host}:#{port}")
|> Keyword.put(:tls_opts, tls_opts)
end
end
@spec parse_proxy(String.t() | tuple() | nil) ::
{tuple, pos_integer()} | {:error, atom()} | nil
def parse_proxy(nil), do: nil
def parse_proxy(proxy) when is_binary(proxy) do
with [host, port] <- String.split(proxy, ":"),
{port, ""} <- Integer.parse(port) do
{:ok, parse_host(host), port}
else
{_, _} ->
Logger.warn("parsing port in proxy fail #{inspect(proxy)}")
{:error, :error_parsing_port_in_proxy}
:error ->
Logger.warn("parsing port in proxy fail #{inspect(proxy)}")
{:error, :error_parsing_port_in_proxy}
_ ->
Logger.warn("parsing proxy fail #{inspect(proxy)}")
{:error, :error_parsing_proxy}
end
end
def parse_proxy(proxy) when is_tuple(proxy) do
with {_type, host, port} <- proxy do
{:ok, parse_host(host), port}
else
_ ->
Logger.warn("parsing proxy fail #{inspect(proxy)}")
{:error, :error_parsing_proxy}
end
end
@spec parse_host(String.t() | tuple()) :: charlist() | atom()
def parse_host(host) when is_atom(host), do: to_charlist(host)
def parse_host(host) when is_binary(host) do
host = to_charlist(host)
case :inet.parse_address(host) do
{:error, :einval} -> host
{:ok, ip} -> ip
end
end
end

59
lib/pleroma/http/http.ex
View File

@ -28,21 +28,44 @@ defmodule Pleroma.HTTP do
"""
def request(method, url, body \\ "", headers \\ [], options \\ []) do
try do
options = process_request_options(options)
adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun
options =
process_request_options(options)
|> process_sni_options(url)
if adapter_gun? do
adapter_opts =
Keyword.get(options, :adapter, [])
|> Keyword.put(:url, url)
Keyword.put(options, :adapter, adapter_opts)
else
options
end
params = Keyword.get(options, :params, [])
%{}
|> Builder.method(method)
|> Builder.headers(headers)
|> Builder.opts(options)
|> Builder.url(url)
|> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
|> Enum.into([])
|> (&Tesla.request(Connection.new(options), &1)).()
request =
%{}
|> Builder.method(method)
|> Builder.url(url)
|> Builder.headers(headers)
|> Builder.opts(options)
|> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
|> Enum.into([])
client = Connection.new(options)
response = Tesla.request(client, request)
if adapter_gun? do
%{adapter: {_, _, [adapter_options]}} = client
pool = adapter_options[:pool]
Pleroma.Gun.Connections.checkout(adapter_options[:conn], self(), pool)
Pleroma.Gun.Connections.process_queue(pool)
end
response
rescue
e ->
{:error, e}
@ -52,20 +75,8 @@ defmodule Pleroma.HTTP do
end
end
defp process_sni_options(options, nil), do: options
defp process_sni_options(options, url) do
uri = URI.parse(url)
host = uri.host |> to_charlist()
case uri.scheme do
"https" -> options ++ [ssl: [server_name_indication: host]]
_ -> options
end
end
def process_request_options(options) do
Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
Keyword.merge(Pleroma.HTTP.Connection.options([]), options)
end
@doc """

2
lib/pleroma/http/request_builder.ex
View File

@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.RequestBuilder do
def headers(request, header_list) do
header_list =
if Pleroma.Config.get([:http, :send_user_agent]) do
header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
header_list ++ [{"user-agent", Pleroma.Application.user_agent()}]
else
header_list
end

6
lib/pleroma/object/fetcher.ex
View File

@ -95,7 +95,7 @@ defmodule Pleroma.Object.Fetcher do
date: date
})
[{:Signature, signature}]
[{"signature", signature}]
end
defp sign_fetch(headers, id, date) do
@ -108,7 +108,7 @@ defmodule Pleroma.Object.Fetcher do
defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
headers ++ [{:Date, date}]
headers ++ [{"date", date}]
else
headers
end
@ -120,7 +120,7 @@ defmodule Pleroma.Object.Fetcher do
date = Pleroma.Signature.signed_date()
headers =
[{:Accept, "application/activity+json"}]
[{"accept", "application/activity+json"}]
|> maybe_date_fetch(date)
|> sign_fetch(id, date)

17
lib/pleroma/reverse_proxy/client.ex
View File

@ -3,9 +3,14 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client do
@callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
{:ok, pos_integer(), [tuple()], reference() | map()}
| {:ok, pos_integer(), [tuple()]}
@type status :: pos_integer()
@type header_name :: String.t()
@type header_value :: String.t()
@type headers :: [{header_name(), header_value()}]
@callback request(atom(), String.t(), headers(), String.t(), list()) ::
{:ok, status(), headers(), reference() | map()}
| {:ok, status(), headers()}
| {:ok, reference()}
| {:error, term()}
@ -14,8 +19,8 @@ defmodule Pleroma.ReverseProxy.Client do
@callback close(reference() | pid() | map()) :: :ok
def request(method, url, headers, "", opts \\ []) do
client().request(method, url, headers, "", opts)
def request(method, url, headers, body \\ "", opts \\ []) do
client().request(method, url, headers, body, opts)
end
def stream_body(ref), do: client().stream_body(ref)
@ -23,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do
def close(ref), do: client().close(ref)
defp client do
Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla)
end
end

60
lib/pleroma/reverse_proxy/client/tesla.ex Normal file
View File

@ -0,0 +1,60 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-onl
defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
@adapters [Tesla.Adapter.Gun]
def request(method, url, headers, body, opts \\ []) do
adapter_opts =
Keyword.get(opts, :adapter, [])
|> Keyword.put(:body_as, :chunks)
with {:ok, response} <-
Pleroma.HTTP.request(
method,
url,
body,
headers,
Keyword.put(opts, :adapter, adapter_opts)
) do
if is_map(response.body),
do: {:ok, response.status, response.headers, response.body},
else: {:ok, response.status, response.headers}
else
{:error, error} -> {:error, error}
end
end
def stream_body(%{fin: true}), do: :done
def stream_body(client) do
case read_chunk!(client) do
{:fin, body} -> {:ok, body, Map.put(client, :fin, true)}
{:nofin, part} -> {:ok, part, client}
{:error, error} -> {:error, error}
end
end
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
adapter = Application.get_env(:tesla, :adapter)
unless adapter in @adapters do
raise "#{adapter} doesn't support reading body in chunks"
end
adapter.read_chunk(pid, stream, opts)
end
def close(pid) do
adapter = Application.get_env(:tesla, :adapter)
unless adapter in @adapters do
raise "#{adapter} doesn't support closing connection"
end
adapter.close(pid)
end
end

18
lib/pleroma/reverse_proxy/reverse_proxy.ex
View File

@ -58,10 +58,10 @@ defmodule Pleroma.ReverseProxy do
* `req_headers`, `resp_headers` additional headers.
* `http`: options for [hackney](https://github.com/benoitc/hackney).
* `http`: options for [gun](https://github.com/ninenines/gun).
"""
@default_hackney_options [pool: :media]
@default_options [pool: :media]
@inline_content_types [
"image/gif",
@ -93,9 +93,9 @@ defmodule Pleroma.ReverseProxy do
def call(_conn, _url, _opts \\ [])
def call(conn = %{method: method}, url, opts) when method in @methods do
hackney_opts =
Pleroma.HTTP.Connection.hackney_options([])
|> Keyword.merge(@default_hackney_options)
client_opts =
Pleroma.HTTP.Connection.options([])
|> Keyword.merge(@default_options)
|> Keyword.merge(Keyword.get(opts, :http, []))
|> HTTP.process_request_options()
@ -108,7 +108,7 @@ defmodule Pleroma.ReverseProxy do
opts
end
with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
with {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
@ -147,11 +147,11 @@ defmodule Pleroma.ReverseProxy do
|> halt()
end
defp request(method, url, headers, hackney_opts) do
defp request(method, url, headers, opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom()
case client().request(method, url, headers, "", hackney_opts) do
case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client}
@ -201,7 +201,7 @@ defmodule Pleroma.ReverseProxy do
duration,
Keyword.get(opts, :max_read_duration, @max_read_duration)
),
{:ok, data} <- client().stream_body(client),
{:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(duration),
sent_so_far = sent_so_far + byte_size(data),
:ok <-

4
lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex
View File

@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger
@hackney_options [
@options [
pool: :media,
recv_timeout: 10_000
]
@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
url
|> MediaProxy.url()
|> HTTP.get([], adapter: @hackney_options)
|> HTTP.get([], adapter: @options)
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

6
lib/pleroma/web/admin_api/config.ex
View File

@ -95,7 +95,6 @@ defmodule Pleroma.Web.AdminAPI.Config do
end
defp do_convert({:dispatch, [entity]}), do: %{"tuple" => [":dispatch", [inspect(entity)]]}
defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
defp do_convert(entity) when is_tuple(entity),
do: %{"tuple" => do_convert(Tuple.to_list(entity))}
@ -129,11 +128,6 @@ defmodule Pleroma.Web.AdminAPI.Config do
{:dispatch, [dispatch_settings]}
end
defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
{partial_chain, []} = do_eval(entity)
{:partial_chain, partial_chain}
end
defp do_transform(%{"tuple" => entity}) do
Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
end

2
lib/pleroma/web/ostatus/ostatus.ex
View File

@ -373,7 +373,7 @@ defmodule Pleroma.Web.OStatus do
{:ok, %{body: body, status: code}} when code in 200..299 <-
HTTP.get(
url,
[{:Accept, "application/atom+xml"}]
[{"accept", "application/atom+xml"}]
) do
Logger.debug("Got document from #{url}, handling...")
handle_incoming(body, options)

4
lib/pleroma/web/rel_me.ex
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RelMe do
@hackney_options [
@options [
pool: :media,
recv_timeout: 2_000,
max_body: 2_000_000,
@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do
def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options)
data =
Floki.attribute(html, "link[rel~=me]", "href") ++

4
lib/pleroma/web/rich_media/parser.ex
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
@hackney_options [
@options [
pool: :media,
recv_timeout: 2_000,
max_body: 2_000_000,
@ -78,7 +78,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
defp parse_url(url) do
try do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options)
html
|> maybe_parse()

2
lib/pleroma/web/web_finger/web_finger.ex
View File

@ -217,7 +217,7 @@ defmodule Pleroma.Web.WebFinger do
with response <-
HTTP.get(
address,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
),
{:ok, %{status: status, body: body}} when status in 200..299 <- response do
doc = XML.parse_document(body)

10
mix.exs
View File

@ -111,7 +111,15 @@ defmodule Pleroma.Mixfile do
{:calendar, "~> 0.17.4"},
{:cachex, "~> 3.0.2"},
{:poison, "~> 3.0", override: true},
{:tesla, "~> 1.2"},
{
:tesla,
github: "alex-strizhakov/tesla",
ref: "199e77f6e4390495eef7c31f2d830da855571b64",
override: true
},
{:cowlib, "~> 2.7.3", override: true},
{:gun,
github: "ninenines/gun", ref: "491ddf58c0e14824a741852fdc522b390b306ae2", override: true},
{:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.1"},

3
mix.lock
View File

@ -37,6 +37,7 @@
"floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
"gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"},
"gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"},
"gun": {:git, "https://github.com/ninenines/gun.git", "491ddf58c0e14824a741852fdc522b390b306ae2", [ref: "491ddf58c0e14824a741852fdc522b390b306ae2"]},
"hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
"html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
"html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
@ -84,7 +85,7 @@
"swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
"tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
"tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "199e77f6e4390495eef7c31f2d830da855571b64", [ref: "199e77f6e4390495eef7c31f2d830da855571b64"]},
"timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
"trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},

685
test/gun/connections_test.exs Normal file
View File

@ -0,0 +1,685 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Gun.ConnectionsTest do
use ExUnit.Case
alias Pleroma.Gun.API
alias Pleroma.Gun.Conn
alias Pleroma.Gun.Connections
setup_all do
{:ok, _} = Registry.start_link(keys: :unique, name: API.Mock)
:ok
end
setup do
name = :test_gun_connections
adapter = Application.get_env(:tesla, :adapter)
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end)
{:ok, pid} = Connections.start_link({name, [max_connections: 2, timeout: 10]})
{:ok, name: name, pid: pid}
end
describe "alive?/2" do
test "is alive", %{name: name} do
assert Connections.alive?(name)
end
test "returns false if not started" do
refute Connections.alive?(:some_random_name)
end
end
test "opens connection and reuse it on next request", %{name: name, pid: pid} do
conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: [],
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name)
assert conn == reused_conn
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: [],
used_by: [{^self, _}, {^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: [],
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: [],
used_by: [],
conn_state: :idle
}
}
} = Connections.get_state(name)
end
test "reuses connection based on protocol", %{name: name, pid: pid} do
conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name)
assert is_pid(conn)
assert Process.alive?(conn)
https_conn = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name)
refute conn == https_conn
reused_https = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name)
refute conn == reused_https
assert reused_https == https_conn
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
},
"https:some-domain.com:443" => %Conn{
conn: ^https_conn,
gun_state: :up,
waiting_pids: []
}
}
} = Connections.get_state(name)
end
test "process gun_down message", %{name: name, pid: pid} do
conn = Connections.checkin("http://gun_down.com", [genserver_pid: pid], name)
refute conn
%Connections{
conns: %{
"http:gun_down.com:80" => %Conn{
conn: _,
gun_state: :down,
waiting_pids: _
}
}
} = Connections.get_state(name)
end
test "process gun_down message and then gun_up", %{name: name, pid: pid} do
conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name)
refute conn
%Connections{
conns: %{
"http:gun_down_and_up.com:80" => %Conn{
conn: _,
gun_state: :down,
waiting_pids: _
}
}
} = Connections.get_state(name)
conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name)
assert is_pid(conn)
assert Process.alive?(conn)
%Connections{
conns: %{
"http:gun_down_and_up.com:80" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
}
}
} = Connections.get_state(name)
end
test "async processes get same conn for same domain", %{name: name, pid: pid} do
tasks =
for _ <- 1..5 do
Task.async(fn ->
Connections.checkin("http://some-domain.com", [genserver_pid: pid], name)
end)
end
tasks_with_results = Task.yield_many(tasks)
results =
Enum.map(tasks_with_results, fn {task, res} ->
res || Task.shutdown(task, :brutal_kill)
end)
conns = for {:ok, value} <- results, do: value
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: conn,
gun_state: :up,
waiting_pids: []
}
}
} = Connections.get_state(name)
assert Enum.all?(conns, fn res -> res == conn end)
end
test "remove frequently used and idle", %{name: name, pid: pid} do
self = self()
conn1 = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name)
[conn2 | _conns] =
for _ <- 1..4 do
Connections.checkin("http://some-domain.com", [genserver_pid: pid], name)
end
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: ^conn2,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}]
},
"https:some-domain.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
:ok = Connections.checkout(conn1, self, name)
conn = Connections.checkin("http://another-domain.com", [genserver_pid: pid], name)
%Connections{
conns: %{
"http:another-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
},
"http:some-domain.com:80" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
end
describe "integration test" do
@describetag :integration
test "opens connection and reuse it on next request", %{name: name} do
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
conn = Connections.checkin("http://httpbin.org", [], name)
assert is_pid(conn)
assert Process.alive?(conn)
reused_conn = Connections.checkin("http://httpbin.org", [], name)
assert conn == reused_conn
%Connections{
conns: %{
"http:httpbin.org:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
}
} = Connections.get_state(name)
end
test "opens ssl connection and reuse it on next request", %{name: name} do
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
conn = Connections.checkin("https://httpbin.org", [], name)
assert is_pid(conn)
assert Process.alive?(conn)
reused_conn = Connections.checkin("https://httpbin.org", [], name)
assert conn == reused_conn
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
}
} = Connections.get_state(name)
end
test "remove frequently used and idle", %{name: name, pid: pid} do
self = self()
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name)
for _ <- 1..4 do
Connections.checkin("https://httpbin.org", [genserver_pid: pid], name)
end
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
},
"https:www.google.com:443" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"http:httpbin.org:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
},
"https:httpbin.org:443" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
end
test "remove earlier used and idle", %{name: name, pid: pid} do
self = self()
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
Connections.checkin("https://www.google.com", [genserver_pid: pid], name)
conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name)
Process.sleep(1_000)
Connections.checkin("https://httpbin.org", [genserver_pid: pid], name)
Connections.checkin("https://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
},
"https:www.google.com:443" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
:ok = Connections.checkout(conn, self, name)
Process.sleep(1_000)
conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"http:httpbin.org:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
},
"https:httpbin.org:443" => %Conn{
conn: _,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
end
test "doesn't drop active connections on pool overflow addinng new requests to the queue", %{
name: name,
pid: pid
} do
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
self = self()
Connections.checkin("https://www.google.com", [genserver_pid: pid], name)
conn1 = Connections.checkin("https://www.google.com", [genserver_pid: pid], name)
conn2 = Connections.checkin("https://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: ^conn2,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
},
"https:www.google.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}, {^self, _}]
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
task =
Task.async(fn -> Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) end)
task_pid = task.pid
:ok = Connections.checkout(conn1, self, name)
Process.sleep(1_000)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: ^conn2,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
},
"https:www.google.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
}
},
queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}],
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
:ok = Connections.checkout(conn1, self, name)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: ^conn2,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
},
"https:www.google.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
waiting_pids: [],
conn_state: :idle,
used_by: []
}
},
queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}],
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
:ok = Connections.process_queue(name)
conn = Task.await(task)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: ^conn2,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^self, _}]
},
"http:httpbin.org:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: [],
conn_state: :active,
used_by: [{^task_pid, _}]
}
},
queue: [],
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
end
end
describe "with proxy usage" do
test "proxy as ip", %{name: name, pid: pid} do
conn =
Connections.checkin(
"http://proxy_string.com",
[genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}],
name
)
%Connections{
conns: %{
"http:proxy_string.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
reused_conn =
Connections.checkin(
"http://proxy_string.com",
[genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}],
name
)
assert reused_conn == conn
end
test "proxy as host", %{name: name, pid: pid} do
conn =
Connections.checkin(
"http://proxy_tuple_atom.com",
[genserver_pid: pid, proxy: {'localhost', 9050}],
name
)
%Connections{
conns: %{
"http:proxy_tuple_atom.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
reused_conn =
Connections.checkin(
"http://proxy_tuple_atom.com",
[genserver_pid: pid, proxy: {'localhost', 9050}],
name
)
assert reused_conn == conn
end
test "proxy as ip and ssl", %{name: name, pid: pid} do
conn =
Connections.checkin(
"https://proxy_string.com",
[genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}],
name
)
%Connections{
conns: %{
"https:proxy_string.com:443" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
reused_conn =
Connections.checkin(
"https://proxy_string.com",
[genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}],
name
)
assert reused_conn == conn
end
test "proxy as host and ssl", %{name: name, pid: pid} do
conn =
Connections.checkin(
"https://proxy_tuple_atom.com",
[genserver_pid: pid, proxy: {'localhost', 9050}],
name
)
%Connections{
conns: %{
"https:proxy_tuple_atom.com:443" => %Conn{
conn: ^conn,
gun_state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
reused_conn =
Connections.checkin(
"https://proxy_tuple_atom.com",
[genserver_pid: pid, proxy: {'localhost', 9050}],
name
)
assert reused_conn == conn
end
end
describe "crf/3" do
setup do
crf = Connections.crf(1, 10, 1)
{:ok, crf: crf}
end
test "more used will have crf higher", %{crf: crf} do
# used 3 times
crf1 = Connections.crf(1, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 > crf2
end
test "recently used will have crf higher on equal references", %{crf: crf} do
# used 4 sec ago
crf1 = Connections.crf(3, 10, crf)
# used 3 sec ago
crf2 = Connections.crf(4, 10, crf)
assert crf1 > crf2
end
test "equal crf on equal reference and time", %{crf: crf} do
# used 2 times
crf1 = Connections.crf(1, 10, crf)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 == crf2
end
test "recently used will have higher crf", %{crf: crf} do
crf1 = Connections.crf(2, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
crf2 = Connections.crf(3, 10, crf)
crf2 = Connections.crf(4, 10, crf2)
assert crf1 > crf2
end
end
end

65
test/http/connection_test.exs Normal file
View File

@ -0,0 +1,65 @@
defmodule Pleroma.HTTP.ConnectionTest do
use ExUnit.Case, async: true
import ExUnit.CaptureLog
alias Pleroma.HTTP.Connection
describe "parse_host/1" do
test "as atom" do
assert Connection.parse_host(:localhost) == 'localhost'
end
test "as string" do
assert Connection.parse_host("localhost.com") == 'localhost.com'
end
test "as string ip" do
assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1}
end
end
describe "parse_proxy/1" do
test "ip with port" do
assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123}
end
test "host with port" do
assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123}
end
test "as tuple" do
assert Connection.parse_proxy({:socks5, :localhost, 9050}) == {:ok, 'localhost', 9050}
end
test "as tuple with string host" do
assert Connection.parse_proxy({:socks5, "localhost", 9050}) == {:ok, 'localhost', 9050}
end
test "ip without port" do
capture_log(fn ->
assert Connection.parse_proxy("127.0.0.1") == {:error, :error_parsing_proxy}
end) =~ "parsing proxy fail \"127.0.0.1\""
end
test "host without port" do
capture_log(fn ->
assert Connection.parse_proxy("localhost") == {:error, :error_parsing_proxy}
end) =~ "parsing proxy fail \"localhost\""
end
test "host with bad port" do
capture_log(fn ->
assert Connection.parse_proxy("localhost:port") == {:error, :error_parsing_port_in_proxy}
end) =~ "parsing port in proxy fail \"localhost:port\""
end
test "as tuple without port" do
capture_log(fn ->
assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :error_parsing_proxy}
end) =~ "parsing proxy fail {:socks5, :localhost}"
end
test "with nil" do
assert Connection.parse_proxy(nil) == nil
end
end
end

4
test/http/request_builder_test.exs
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.RequestBuilderTest do
use ExUnit.Case, async: true
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.HTTP.RequestBuilder
@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do
Pleroma.Config.put([:http, :send_user_agent], true)
assert RequestBuilder.headers(%{}, []) == %{
headers: [{"User-Agent", Pleroma.Application.user_agent()}]
headers: [{"user-agent", Pleroma.Application.user_agent()}]
}
end
end

29
test/http_test.exs
View File

@ -56,4 +56,33 @@ defmodule Pleroma.HTTPTest do
}
end
end
@tag :integration
test "get_conn_for_gun/3" do
adapter = Application.get_env(:tesla, :adapter)
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
api = Pleroma.Config.get([Pleroma.Gun.API])
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
on_exit(fn ->
Application.put_env(:tesla, :adapter, adapter)
Pleroma.Config.put([Pleroma.Gun.API], api)
end)
options = [adapter: [pool: :federation]]
assert {:ok, resp} = Pleroma.HTTP.get("https://httpbin.org/user-agent", [], options)
adapter_opts = resp.opts[:adapter]
assert resp.status == 200
assert adapter_opts[:url] == "https://httpbin.org/user-agent"
state = Pleroma.Gun.Connections.get_state(:federation)
conn = state.conns["https:httpbin.org:443"]
assert conn.conn_state == :idle
assert conn.used_by == []
assert state.queue == []
end
end

25
test/reverse_proxy/client/tesla_test.exs Normal file
View File

@ -0,0 +1,25 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.TeslaTest do
use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla
setup_all do
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
on_exit(fn ->
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Mock)
end)
end
defp check_ref(%{pid: pid, stream: stream} = ref) do
assert is_pid(pid)
assert is_reference(stream)
assert ref[:fin]
end
defp close(%{pid: pid}) do
Pleroma.ReverseProxy.Client.Tesla.close(pid)
end
end

147
test/reverse_proxy_test.exs → test/reverse_proxy/reverse_proxy_test.exs
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxyTest do
use Pleroma.Web.ConnCase, async: true
use Pleroma.Web.ConnCase
import ExUnit.CaptureLog
import Mox
alias Pleroma.ReverseProxy
@ -29,11 +29,11 @@ defmodule Pleroma.ReverseProxyTest do
{"content-length", byte_size(json) |> to_string()}
], %{url: url}}
end)
|> expect(:stream_body, invokes, fn %{url: url} ->
|> expect(:stream_body, invokes, fn %{url: url} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do
[{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1))
{:ok, json}
{:ok, json, client}
[{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, url)
@ -66,6 +66,38 @@ defmodule Pleroma.ReverseProxyTest do
assert conn.halted
end
defp stream_mock(invokes, with_close? \\ false) do
ClientMock
|> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0)
{:ok, 200, [{"content-type", "application/octet-stream"}],
%{url: "/stream-bytes/" <> length}}
end)
|> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client ->
max = String.to_integer(length)
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do
[{_, current}] when current < max ->
Registry.update_value(
Pleroma.ReverseProxy.ClientMock,
"/stream-bytes/" <> length,
&(&1 + 10)
)
{:ok, "0123456789", client}
[{_, ^max}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length)
:done
end
end)
if with_close? do
expect(ClientMock, :close, fn _ -> :ok end)
end
end
describe "max_body " do
test "length returns error if content-length more than option", %{conn: conn} do
user_agent_mock("hackney/1.15.1", 0)
@ -76,38 +108,6 @@ defmodule Pleroma.ReverseProxyTest do
"[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large"
end
defp stream_mock(invokes, with_close? \\ false) do
ClientMock
|> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0)
{:ok, 200, [{"content-type", "application/octet-stream"}],
%{url: "/stream-bytes/" <> length}}
end)
|> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} ->
max = String.to_integer(length)
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do
[{_, current}] when current < max ->
Registry.update_value(
Pleroma.ReverseProxy.ClientMock,
"/stream-bytes/" <> length,
&(&1 + 10)
)
{:ok, "0123456789"}
[{_, ^max}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length)
:done
end
end)
if with_close? do
expect(ClientMock, :close, fn _ -> :ok end)
end
end
test "max_body_length returns error if streaming body more than that option", %{conn: conn} do
stream_mock(3, true)
@ -179,12 +179,12 @@ defmodule Pleroma.ReverseProxyTest do
Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0)
{:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}}
end)
|> expect(:stream_body, 2, fn %{url: url, headers: headers} ->
|> expect(:stream_body, 2, fn %{url: url, headers: headers} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do
[{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1))
headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v}
{:ok, Jason.encode!(%{headers: headers})}
{:ok, Jason.encode!(%{headers: headers}), client}
[{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, url)
@ -261,11 +261,11 @@ defmodule Pleroma.ReverseProxyTest do
{:ok, 200, headers, %{url: "/disposition"}}
end)
|> expect(:stream_body, 2, fn %{url: "/disposition"} ->
|> expect(:stream_body, 2, fn %{url: "/disposition"} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do
[{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1))
{:ok, ""}
{:ok, "", client}
[{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition")
@ -297,4 +297,73 @@ defmodule Pleroma.ReverseProxyTest do
assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers
end
end
describe "integration tests" do
@describetag :integration
test "with tesla client with gun adapter", %{conn: conn} do
client = Pleroma.Config.get([Pleroma.ReverseProxy.Client])
Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla)
adapter = Application.get_env(:tesla, :adapter)
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
api = Pleroma.Config.get([Pleroma.Gun.API])
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10")
assert byte_size(conn.resp_body) == 10
assert conn.state == :chunked
assert conn.status == 200
on_exit(fn ->
Pleroma.Config.put([Pleroma.ReverseProxy.Client], client)
Application.put_env(:tesla, :adapter, adapter)
Pleroma.Config.put([Pleroma.Gun.API], api)
end)
end
test "with tesla client with gun adapter with ssl", %{conn: conn} do
client = Pleroma.Config.get([Pleroma.ReverseProxy.Client])
Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla)
adapter = Application.get_env(:tesla, :adapter)
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
api = Pleroma.Config.get([Pleroma.Gun.API])
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10")
assert byte_size(conn.resp_body) == 10
assert conn.state == :chunked
assert conn.status == 200
on_exit(fn ->
Pleroma.Config.put([Pleroma.ReverseProxy.Client], client)
Application.put_env(:tesla, :adapter, adapter)
Pleroma.Config.put([Pleroma.Gun.API], api)
end)
end
test "tesla client with gun client follow redirects", %{conn: conn} do
client = Pleroma.Config.get([Pleroma.ReverseProxy.Client])
Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla)
adapter = Application.get_env(:tesla, :adapter)
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
api = Pleroma.Config.get([Pleroma.Gun.API])
Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun)
conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5")
assert conn.state == :chunked
assert conn.status == 200
on_exit(fn ->
Pleroma.Config.put([Pleroma.ReverseProxy.Client], client)
Application.put_env(:tesla, :adapter, adapter)
Pleroma.Config.put([Pleroma.Gun.API], api)
end)
end
end
end

84
test/support/http_request_mock.ex
View File

@ -91,7 +91,7 @@ defmodule HttpRequestMock do
"https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -104,7 +104,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -125,7 +125,7 @@ defmodule HttpRequestMock do
"https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -151,7 +151,7 @@ defmodule HttpRequestMock do
"https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -172,7 +172,7 @@ defmodule HttpRequestMock do
"https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -185,7 +185,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -198,7 +198,7 @@ defmodule HttpRequestMock do
"https://squeet.me/xrd/?uri=lain@squeet.me",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -211,7 +211,7 @@ defmodule HttpRequestMock do
"https://mst3k.interlinked.me/users/luciferMysticus",
_,
_,
Accept: "application/activity+json"
[{"accept", "application/activity+json"}]
) do
{:ok,
%Tesla.Env{
@ -232,7 +232,7 @@ defmodule HttpRequestMock do
"https://hubzilla.example.org/channel/kaniini",
_,
_,
Accept: "application/activity+json"
[{"accept", "application/activity+json"}]
) do
{:ok,
%Tesla.Env{
@ -241,7 +241,7 @@ defmodule HttpRequestMock do
}}
end
def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do
def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
{:ok,
%Tesla.Env{
status: 200,
@ -249,7 +249,7 @@ defmodule HttpRequestMock do
}}
end
def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do
def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
{:ok,
%Tesla.Env{
status: 200,
@ -268,7 +268,7 @@ defmodule HttpRequestMock do
}}
end
def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do
def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do
{:ok,
%Tesla.Env{
status: 200,
@ -324,7 +324,9 @@ defmodule HttpRequestMock do
}}
end
def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do
def get("http://mastodon.example.org/users/admin", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok,
%Tesla.Env{
status: 200,
@ -332,7 +334,9 @@ defmodule HttpRequestMock do
}}
end
def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do
def get("http://mastodon.example.org/users/gargron", _, _, [
{"accept", "application/activity+json"}
]) do
{:error, :nxdomain}
end
@ -340,7 +344,7 @@ defmodule HttpRequestMock do
"http://mastodon.example.org/@admin/99541947525187367",
_,
_,
Accept: "application/activity+json"
[{"accept", "application/activity+json"}]
) do
{:ok,
%Tesla.Env{
@ -357,7 +361,7 @@ defmodule HttpRequestMock do
}}
end
def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do
def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do
{:ok,
%Tesla.Env{
status: 200,
@ -369,7 +373,7 @@ defmodule HttpRequestMock do
"https://mstdn.io/users/mayuutann/statuses/99568293732299394",
_,
_,
Accept: "application/activity+json"
[{"accept", "application/activity+json"}]
) do
{:ok,
%Tesla.Env{
@ -389,7 +393,7 @@ defmodule HttpRequestMock do
}}
end
def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json")
def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}])
when url in [
"https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain",
"https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain"
@ -416,7 +420,7 @@ defmodule HttpRequestMock do
"https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -460,7 +464,7 @@ defmodule HttpRequestMock do
"https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -513,7 +517,7 @@ defmodule HttpRequestMock do
"https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -526,7 +530,7 @@ defmodule HttpRequestMock do
"https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056",
_,
_,
Accept: "application/atom+xml"
[{"accept", "application/atom+xml"}]
) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}}
end
@ -543,7 +547,7 @@ defmodule HttpRequestMock do
"https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -565,7 +569,7 @@ defmodule HttpRequestMock do
"http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -579,7 +583,7 @@ defmodule HttpRequestMock do
"http://gs.example.org:4040/index.php/user/1",
_,
_,
Accept: "application/activity+json"
[{"accept", "application/activity+json"}]
) do
{:ok, %Tesla.Env{status: 406, body: ""}}
end
@ -615,7 +619,7 @@ defmodule HttpRequestMock do
"https://squeet.me/xrd?uri=lain@squeet.me",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -628,7 +632,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -641,7 +645,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok, %Tesla.Env{status: 200, body: ""}}
end
@ -658,7 +662,7 @@ defmodule HttpRequestMock do
"http://framatube.org/main/xrd?uri=framasoft@framatube.org",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -680,7 +684,7 @@ defmodule HttpRequestMock do
"http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -717,7 +721,7 @@ defmodule HttpRequestMock do
"https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -775,7 +779,7 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}}
end
def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do
def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do
{:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)}
end
@ -892,7 +896,7 @@ defmodule HttpRequestMock do
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -905,7 +909,7 @@ defmodule HttpRequestMock do
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain",
_,
_,
Accept: "application/xrd+xml,application/jrd+json"
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
{:ok,
%Tesla.Env{
@ -927,7 +931,9 @@ defmodule HttpRequestMock do
}}
end
def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do
def get("https://info.pleroma.site/activity.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok,
%Tesla.Env{
status: 200,
@ -939,7 +945,9 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 404, body: ""}}
end
def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do
def get("https://info.pleroma.site/activity2.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok,
%Tesla.Env{
status: 200,
@ -951,7 +959,9 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 404, body: ""}}
end
def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do
def get("https://info.pleroma.site/activity3.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok,
%Tesla.Env{
status: 200,

80
test/support/reverse_proxy_client_case.ex Normal file
View File

@ -0,0 +1,80 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxyClientCase do
defmacro __using__(client: client) do
quote do
use ExUnit.Case
@moduletag :integration
@client unquote(client)
setup do
Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun)
on_exit(fn -> Application.put_env(:tesla, :adapter, Tesla.Mock) end)
end
test "get response body stream" do
{:ok, status, headers, ref} =
@client.request(
:get,
"http://httpbin.org/stream-bytes/10",
[{"accept", "application/octet-stream"}],
"",
[]
)
assert status == 200
assert headers != []
{:ok, response, ref} = @client.stream_body(ref)
check_ref(ref)
assert is_binary(response)
assert byte_size(response) == 10
assert :done == @client.stream_body(ref)
end
test "head response" do
{:ok, status, headers} = @client.request(:head, "http://httpbin.org/get", [], "", [])
assert status == 200
assert headers != []
end
test "get error response" do
case @client.request(
:get,
"http://httpbin.org/status/500",
[],
"",
[]
) do
{:ok, status, headers, ref} ->
assert status == 500
assert headers != []
check_ref(ref)
assert :ok == close(ref)
{:ok, status, headers} ->
assert headers != []
end
end
test "head error response" do
{:ok, status, headers} =
@client.request(
:head,
"http://httpbin.org/status/500",
[],
"",
[]
)
assert status == 500
assert headers != []
end
end
end
end

6
test/web/admin_api/admin_api_controller_test.exs
View File

@ -1779,8 +1779,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
%{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]},
%{"tuple" => [":seconds_valid", 60]},
%{"tuple" => [":path", ""]},
%{"tuple" => [":key1", nil]},
%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}
%{"tuple" => [":key1", nil]}
]
}
]
@ -1796,8 +1795,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
%{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]},
%{"tuple" => [":seconds_valid", 60]},
%{"tuple" => [":path", ""]},
%{"tuple" => [":key1", nil]},
%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}
%{"tuple" => [":key1", nil]}
]
}
]

8
test/web/admin_api/config_test.exs
View File

@ -238,14 +238,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigTest do
assert Config.from_binary(binary) == [key: "value"]
end
test "keyword with partial_chain key" do
binary =
Config.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}])
assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1)
assert Config.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1]
end
test "keyword" do
binary =
Config.transform([

16
test/web/twitter_api/util_controller_test.exs
View File

@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
import Mock
import ExUnit.CaptureLog
setup do
Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@ -367,15 +368,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
assert html_response(response, 200) =~ "Remote follow"
end
test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do
test "show follow page with error when user cannot fetch by `acct` link", %{conn: conn} do
user = insert(:user)
response =
conn
|> assign(:user, user)
|> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found")
assert capture_log(fn ->
response =
conn
|> assign(:user, user)
|> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found")
assert html_response(response, 200) =~ "Error fetching user"
assert html_response(response, 200) =~ "Error fetching user"
end) =~
"Could not decode user at fetch https://mastodon.social/users/not_found, {:error, \"Object has been deleted\"}"
end
end

13
test/web/web_finger/web_finger_controller_test.exs
View File

@ -7,6 +7,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
import Pleroma.Factory
import Tesla.Mock
import ExUnit.CaptureLog
setup do
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@ -75,11 +76,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do
test "Sends a 404 when invalid format" do
user = insert(:user)
assert_raise Phoenix.NotAcceptableError, fn ->
build_conn()
|> put_req_header("accept", "text/html")
|> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost")
end
assert capture_log(fn ->
assert_raise Phoenix.NotAcceptableError, fn ->
build_conn()
|> put_req_header("accept", "text/html")
|> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost")
end
end) =~ "Internal server error:"
end
test "Sends a 400 when resource param is missing" do