squeegily/pleroma
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added deleted_urls_cache

Browse Source
This commit is contained in:
Maksim Pechnikov 2020-05-25 20:55:42 +03:00
parent 755bf36437
commit 6e9e21d6fc
9 changed files with 72 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/pleroma/application.ex
View File

@ -148,7 +148,8 @@ defmodule Pleroma.Application do
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500)
build_cachex("failed_proxy_url", limit: 2500),
build_cachex("deleted_urls", default_ttl: :timer.minutes(60), limit: 2500)
]
end

25
lib/pleroma/web/media_proxy/invalidation.ex
View File

@ -5,27 +5,32 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false
@callback purge(list(String.t()), Keyword.t()) :: {:ok, String.t()} | {:error, String.t()}
@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config
alias Pleroma.Web.MediaProxy
@spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
@spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do
[:media_proxy, :invalidation, :enabled]
|> Config.get()
|> do_purge(urls)
prepared_urls = prepare_urls(urls)
if Config.get([:media_proxy, :invalidation, :enabled]) do
result = do_purge(prepared_urls)
MediaProxy.remove_from_deleted_urls(prepared_urls)
result
else
{:ok, prepared_urls}
end
end
defp do_purge(true, urls) do
defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider])
options = Config.get(provider)
provider.purge(urls, Config.get(provider))
end
defp prepare_urls(urls) do
urls
|> List.wrap()
|> Enum.map(&MediaProxy.url(&1))
|> provider.purge(options)
end
defp do_purge(_, _), do: :ok
end

2
lib/pleroma/web/media_proxy/invalidations/http.ex
View File

@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
end
end)
{:ok, "success"}
{:ok, urls}
end
defp do_purge(method, url, headers, options) do

8
lib/pleroma/web/media_proxy/invalidations/script.ex
View File

@ -20,7 +20,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
opts
|> Keyword.get(:script_path, nil)
|> do_purge([args])
|> handle_error
|> handle_result(urls)
end
defp do_purge(script_path, args) when is_binary(script_path) do
@ -33,10 +33,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
defp do_purge(_, _), do: {:error, "not found script path"}
defp handle_error({_result, 0}), do: {:ok, "success"}
defp handle_error({:error, error}), do: handle_error(error)
defp handle_result({_result, 0}, urls), do: {:ok, urls}
defp handle_result({:error, error}, urls), do: handle_result(error, urls)
defp handle_error(error) do
defp handle_result(error, _) do
Logger.error("Error while cache purge: #{inspect(error)}")
{:error, inspect(error)}
end

19
lib/pleroma/web/media_proxy/media_proxy.ex
View File

@ -9,6 +9,25 @@ defmodule Pleroma.Web.MediaProxy do
@base64_opts [padding: false]
@spec in_deleted_urls(String.t()) :: boolean()
def in_deleted_urls(url), do: !!Cachex.get!(:deleted_urls_cache, url)
def remove_from_deleted_urls(urls) when is_list(urls) do
Enum.each(urls, &remove_from_deleted_urls(&1))
end
def remove_from_deleted_urls(url) when is_binary(url) do
Cachex.del(:deleted_urls_cache, url)
end
def put_in_deleted_urls(urls) when is_list(urls) do
Enum.each(urls, &put_in_deleted_urls(&1))
end
def put_in_deleted_urls(url) when is_binary(url) do
Cachex.put(:deleted_urls_cache, url, true)
end
def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url

3
lib/pleroma/web/media_proxy/media_proxy_controller.ex
View File

@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
{_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
error when error in [false, {:in_deleted_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->

21
test/web/media_proxy/invalidation_test.exs
View File

@ -13,10 +13,13 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
describe "Invalidation.Http" do
test "perform request to clear cache" do
Config.put([:media_proxy, :enabled], false)
Config.put([:media_proxy, :invalidation, :enabled], true)
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
image_url = "http://example.com/media/example.jpg"
Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
mock(fn
%{
@ -28,23 +31,29 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
end)
assert capture_log(fn ->
assert Invalidation.purge(["http://example.com/media/example.jpg"]) ==
{:ok, "success"}
end) =~ "Running cache purge: [\"http://example.com/media/example.jpg\"]"
assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
end) =~ "Running cache purge: [\"#{image_url}\"]"
end
end
describe "Invalidation.Script" do
test "run script to clear cache" do
Config.put([:media_proxy, :enabled], false)
Config.put([:media_proxy, :invalidation, :enabled], true)
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
Config.put([Invalidation.Script], script_path: "purge-nginx")
image_url = "http://example.com/media/example.jpg"
Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
assert capture_log(fn ->
assert Invalidation.purge(["http://example.com/media/example.jpg"]) ==
{:ok, "success"}
end) =~ "Running cache purge: [\"http://example.com/media/example.jpg\"]"
assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
end) =~ "Running cache purge: [\"#{image_url}\"]"
end
end
end

4
test/web/media_proxy/invalidations/http_test.exs
View File

@ -15,7 +15,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
assert Invalidation.Http.purge(
["http://example.com/media/example.jpg"],
[]
) == {:ok, "success"}
) == {:ok, ["http://example.com/media/example.jpg"]}
end) =~ "Error while cache purge"
end
@ -29,7 +29,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
assert Invalidation.Http.purge(
["http://example.com/media/example1.jpg"],
[]
) == {:ok, "success"}
) == {:ok, ["http://example.com/media/example1.jpg"]}
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
end
end

12
test/web/media_proxy/media_proxy_controller_test.exs
View File

@ -66,4 +66,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
assert %Plug.Conn{status: :success} = get(conn, url)
end
end
test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do
Config.put([:media_proxy, :enabled], true)
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png")
with_mock Pleroma.ReverseProxy,
call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
end
end
end