This shouldn't be too expensive, since the connections are pooled, but it should save us some bandwidth since we won't fetch non-html files and files that are too large for us to process (especially since you can't cancel a request without closing the connection with HTTP1).merge-requests/3010/head
@@ -87,6 +87,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do | |||||
def rich_media_get(url) do | def rich_media_get(url) do | ||||
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] | headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] | ||||
Pleroma.HTTP.get(url, headers, @options) | |||||
head_check = | |||||
case Pleroma.HTTP.head(url, headers, @options) do | |||||
# If the HEAD request didn't reach the server for whatever reason, | |||||
# we assume the GET that comes right after won't either | |||||
{:error, _} = e -> | |||||
e | |||||
{:ok, %Tesla.Env{status: 200, headers: headers}} -> | |||||
with :ok <- check_content_type(headers), | |||||
:ok <- check_content_length(headers), | |||||
do: :ok | |||||
_ -> | |||||
:ok | |||||
end | |||||
with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options) | |||||
end | |||||
defp check_content_type(headers) do | |||||
case List.keyfind(headers, "content-type", 0) do | |||||
{_, content_type} -> | |||||
case Plug.Conn.Utils.media_type(content_type) do | |||||
{:ok, "text", "html", _} -> :ok | |||||
_ -> {:error, {:content_type, content_type}} | |||||
end | |||||
_ -> | |||||
:ok | |||||
end | |||||
end | |||||
@max_body @options[:max_body] | |||||
defp check_content_length(headers) do | |||||
case List.keyfind(headers, "content-length", 0) do | |||||
{_, maybe_content_length} -> | |||||
case Integer.parse(maybe_content_length) do | |||||
{content_length, ""} when content_length <= @max_body -> :ok | |||||
{_, ""} -> {:error, :body_too_large} | |||||
_ -> :ok | |||||
end | |||||
_ -> | |||||
:ok | |||||
end | |||||
end | end | ||||
end | end |
@@ -1262,4 +1262,21 @@ defmodule HttpRequestMock do | |||||
inspect(headers) | inspect(headers) | ||||
}"} | }"} | ||||
end | end | ||||
# Most of the rich media mocks are missing HEAD requests, so we just return 404. | |||||
@rich_media_mocks [ | |||||
"https://example.com/ogp", | |||||
"https://example.com/ogp-missing-data", | |||||
"https://example.com/twitter-card" | |||||
] | |||||
def head(url, _query, _body, _headers) when url in @rich_media_mocks do | |||||
{:ok, %Tesla.Env{status: 404, body: ""}} | |||||
end | |||||
def head(url, query, body, headers) do | |||||
{:error, | |||||
"Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{ | |||||
inspect(headers) | |||||
}"} | |||||
end | |||||
end | end |
@@ -56,6 +56,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do | |||||
%{method: :get, url: "http://example.com/error"} -> | %{method: :get, url: "http://example.com/error"} -> | ||||
{:error, :overload} | {:error, :overload} | ||||
%{ | |||||
method: :head, | |||||
url: "http://example.com/huge-page" | |||||
} -> | |||||
%Tesla.Env{ | |||||
status: 200, | |||||
headers: [{"content-length", "2000001"}, {"content-type", "text/html"}] | |||||
} | |||||
%{ | |||||
method: :head, | |||||
url: "http://example.com/pdf-file" | |||||
} -> | |||||
%Tesla.Env{ | |||||
status: 200, | |||||
headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}] | |||||
} | |||||
%{method: :head} -> | |||||
%Tesla.Env{status: 404, body: "", headers: []} | |||||
end) | end) | ||||
:ok | :ok | ||||
@@ -144,4 +165,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do | |||||
test "returns error if getting page was not successful" do | test "returns error if getting page was not successful" do | ||||
assert {:error, :overload} = Parser.parse("http://example.com/error") | assert {:error, :overload} = Parser.parse("http://example.com/error") | ||||
end | end | ||||
test "does a HEAD request to check if the body is too large" do | |||||
assert {:error, body_too_large} = Parser.parse("http://example.com/huge-page") | |||||
end | |||||
test "does a HEAD request to check if the body is html" do | |||||
assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file") | |||||
end | |||||
end | end |