Add Pleroma.Plugs.Cache Closes #1174 See merge request pleroma/pleroma!1612tags/v1.1.4
@@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). | |||
- Mix Tasks: `mix pleroma.database fix_likes_collections` | |||
- Federation: Remove `likes` from objects. | |||
- Admin API: Added moderation log | |||
- Web response cache (currently, enabled for ActivityPub) | |||
### Changed | |||
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text | |||
@@ -560,6 +560,10 @@ config :pleroma, :rate_limit, nil | |||
config :pleroma, Pleroma.ActivityExpiration, enabled: true | |||
config :pleroma, :web_cache_ttl, | |||
activity_pub: nil, | |||
activity_pub_question: 30_000 | |||
# Import environment specific config. This must remain at the bottom | |||
# of this file so it overrides the configuration defined above. | |||
import_config "#{Mix.env()}.exs" |
@@ -690,3 +690,12 @@ Supported rate limiters: | |||
* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) | |||
* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses | |||
* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user | |||
## :web_cache_ttl | |||
The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. | |||
Available caches: | |||
* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). | |||
* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). |
@@ -308,10 +308,19 @@ defmodule Pleroma.Activity do | |||
%{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id | |||
_ -> nil | |||
end) | |||
|> purge_web_resp_cache() | |||
end | |||
def delete_by_ap_id(_), do: nil | |||
defp purge_web_resp_cache(%Activity{} = activity) do | |||
%{path: path} = URI.parse(activity.data["id"]) | |||
Cachex.del(:web_resp_cache, path) | |||
activity | |||
end | |||
defp purge_web_resp_cache(nil), do: nil | |||
for {ap_type, type} <- @mastodon_notification_types do | |||
def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}), | |||
do: unquote(type) | |||
@@ -116,7 +116,8 @@ defmodule Pleroma.Application do | |||
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), | |||
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), | |||
build_cachex("scrubber", limit: 2500), | |||
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500) | |||
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), | |||
build_cachex("web_resp", limit: 2500) | |||
] | |||
end | |||
@@ -130,14 +130,16 @@ defmodule Pleroma.Object do | |||
def delete(%Object{data: %{"id" => id}} = object) do | |||
with {:ok, _obj} = swap_object_with_tombstone(object), | |||
deleted_activity = Activity.delete_by_ap_id(id), | |||
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do | |||
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), | |||
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do | |||
{:ok, object, deleted_activity} | |||
end | |||
end | |||
def prune(%Object{data: %{"id" => id}} = object) do | |||
with {:ok, object} <- Repo.delete(object), | |||
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do | |||
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), | |||
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do | |||
{:ok, object} | |||
end | |||
end | |||
@@ -0,0 +1,122 @@ | |||
# Pleroma: A lightweight social networking server | |||
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> | |||
# SPDX-License-Identifier: AGPL-3.0-only | |||
defmodule Pleroma.Plugs.Cache do | |||
@moduledoc """ | |||
Caches successful GET responses. | |||
To enable the cache add the plug to a router pipeline or controller: | |||
plug(Pleroma.Plugs.Cache) | |||
## Configuration | |||
To configure the plug you need to pass settings as the second argument to the `plug/2` macro: | |||
plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true]) | |||
Available options: | |||
- `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. | |||
- `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. | |||
Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: | |||
def index(conn, _params) do | |||
ttl = 60_000 # one minute | |||
conn | |||
|> assign(:cache_ttl, ttl) | |||
|> render("index.html") | |||
end | |||
""" | |||
import Phoenix.Controller, only: [current_path: 1, json: 2] | |||
import Plug.Conn | |||
@behaviour Plug | |||
@defaults %{ttl: nil, query_params: true} | |||
@impl true | |||
def init([]), do: @defaults | |||
def init(opts) do | |||
opts = Map.new(opts) | |||
Map.merge(@defaults, opts) | |||
end | |||
@impl true | |||
def call(%{method: "GET"} = conn, opts) do | |||
key = cache_key(conn, opts) | |||
case Cachex.get(:web_resp_cache, key) do | |||
{:ok, nil} -> | |||
cache_resp(conn, opts) | |||
{:ok, record} -> | |||
send_cached(conn, record) | |||
{atom, message} when atom in [:ignore, :error] -> | |||
render_error(conn, message) | |||
end | |||
end | |||
def call(conn, _), do: conn | |||
# full path including query params | |||
defp cache_key(conn, %{query_params: true}), do: current_path(conn) | |||
# request path without query params | |||
defp cache_key(conn, %{query_params: false}), do: conn.request_path | |||
# request path with specific query params | |||
defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do | |||
query_string = | |||
conn.params | |||
|> Map.take(query_params) | |||
|> URI.encode_query() | |||
conn.request_path <> "?" <> query_string | |||
end | |||
defp cache_resp(conn, opts) do | |||
register_before_send(conn, fn | |||
%{status: 200, resp_body: body} = conn -> | |||
ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl) | |||
key = cache_key(conn, opts) | |||
content_type = content_type(conn) | |||
record = {content_type, body} | |||
Cachex.put(:web_resp_cache, key, record, ttl: ttl) | |||
put_resp_header(conn, "x-cache", "MISS from Pleroma") | |||
conn -> | |||
conn | |||
end) | |||
end | |||
defp content_type(conn) do | |||
conn | |||
|> Plug.Conn.get_resp_header("content-type") | |||
|> hd() | |||
end | |||
defp send_cached(conn, {content_type, body}) do | |||
conn | |||
|> put_resp_content_type(content_type, nil) | |||
|> put_resp_header("x-cache", "HIT from Pleroma") | |||
|> send_resp(:ok, body) | |||
|> halt() | |||
end | |||
defp render_error(conn, message) do | |||
conn | |||
|> put_status(:internal_server_error) | |||
|> json(%{error: message}) | |||
|> halt() | |||
end | |||
end |
@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
action_fallback(:errors) | |||
plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object]) | |||
plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) | |||
plug(:set_requester_reachable when action in [:inbox]) | |||
plug(:relay_active? when action in [:relay]) | |||
@@ -53,8 +54,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
%Object{} = object <- Object.get_cached_by_ap_id(ap_id), | |||
{_, true} <- {:public?, Visibility.is_public?(object)} do | |||
conn | |||
|> set_cache_ttl_for(object) | |||
|> put_resp_content_type("application/activity+json") | |||
|> json(ObjectView.render("object.json", %{object: object})) | |||
|> put_view(ObjectView) | |||
|> render("object.json", object: object) | |||
else | |||
{:public?, false} -> | |||
{:error, :not_found} | |||
@@ -96,14 +99,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
%Activity{} = activity <- Activity.normalize(ap_id), | |||
{_, true} <- {:public?, Visibility.is_public?(activity)} do | |||
conn | |||
|> set_cache_ttl_for(activity) | |||
|> put_resp_content_type("application/activity+json") | |||
|> json(ObjectView.render("object.json", %{object: activity})) | |||
|> put_view(ObjectView) | |||
|> render("object.json", object: activity) | |||
else | |||
{:public?, false} -> | |||
{:error, :not_found} | |||
{:public?, false} -> {:error, :not_found} | |||
nil -> {:error, :not_found} | |||
end | |||
end | |||
defp set_cache_ttl_for(conn, %Activity{object: object}) do | |||
set_cache_ttl_for(conn, object) | |||
end | |||
defp set_cache_ttl_for(conn, entity) do | |||
ttl = | |||
case entity do | |||
%Object{data: %{"type" => "Question"}} -> | |||
Pleroma.Config.get([:web_cache_ttl, :activity_pub_question]) | |||
%Object{} -> | |||
Pleroma.Config.get([:web_cache_ttl, :activity_pub]) | |||
_ -> | |||
nil | |||
end | |||
assign(conn, :cache_ttl, ttl) | |||
end | |||
# GET /relay/following | |||
def following(%{assigns: %{relay: true}} = conn, _params) do | |||
conn | |||
@@ -53,9 +53,12 @@ defmodule Pleroma.ObjectTest do | |||
assert object == cached_object | |||
Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe") | |||
Object.delete(cached_object) | |||
{:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}") | |||
{:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path) | |||
cached_object = Object.get_cached_by_ap_id(object.data["id"]) | |||
@@ -0,0 +1,186 @@ | |||
# Pleroma: A lightweight social networking server | |||
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> | |||
# SPDX-License-Identifier: AGPL-3.0-only | |||
defmodule Pleroma.Plugs.CacheTest do | |||
use ExUnit.Case, async: true | |||
use Plug.Test | |||
alias Pleroma.Plugs.Cache | |||
@miss_resp {200, | |||
[ | |||
{"cache-control", "max-age=0, private, must-revalidate"}, | |||
{"content-type", "cofe/hot; charset=utf-8"}, | |||
{"x-cache", "MISS from Pleroma"} | |||
], "cofe"} | |||
@hit_resp {200, | |||
[ | |||
{"cache-control", "max-age=0, private, must-revalidate"}, | |||
{"content-type", "cofe/hot; charset=utf-8"}, | |||
{"x-cache", "HIT from Pleroma"} | |||
], "cofe"} | |||
@ttl 5 | |||
setup do | |||
Cachex.clear(:web_resp_cache) | |||
:ok | |||
end | |||
test "caches a response" do | |||
assert @miss_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert_raise(Plug.Conn.AlreadySentError, fn -> | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end) | |||
assert @hit_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> sent_resp() | |||
end | |||
test "ttl is set" do | |||
assert @miss_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: @ttl}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert @hit_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: @ttl}) | |||
|> sent_resp() | |||
:timer.sleep(@ttl + 1) | |||
assert @miss_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: @ttl}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end | |||
test "set ttl via conn.assigns" do | |||
assert @miss_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> assign(:cache_ttl, @ttl) | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert @hit_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> sent_resp() | |||
:timer.sleep(@ttl + 1) | |||
assert @miss_resp == | |||
conn(:get, "/") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end | |||
test "ignore query string when `query_params` is false" do | |||
assert @miss_resp == | |||
conn(:get, "/?cofe") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert @hit_resp == | |||
conn(:get, "/?cofefe") | |||
|> Cache.call(%{query_params: false, ttl: nil}) | |||
|> sent_resp() | |||
end | |||
test "take query string into account when `query_params` is true" do | |||
assert @miss_resp == | |||
conn(:get, "/?cofe") | |||
|> Cache.call(%{query_params: true, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert @miss_resp == | |||
conn(:get, "/?cofefe") | |||
|> Cache.call(%{query_params: true, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end | |||
test "take specific query params into account when `query_params` is list" do | |||
assert @miss_resp == | |||
conn(:get, "/?a=1&b=2&c=3&foo=bar") | |||
|> fetch_query_params() | |||
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
assert @hit_resp == | |||
conn(:get, "/?bar=foo&c=3&b=2&a=1") | |||
|> fetch_query_params() | |||
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) | |||
|> sent_resp() | |||
assert @miss_resp == | |||
conn(:get, "/?bar=foo&c=3&b=2&a=2") | |||
|> fetch_query_params() | |||
|> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end | |||
test "ignore not GET requests" do | |||
expected = | |||
{200, | |||
[ | |||
{"cache-control", "max-age=0, private, must-revalidate"}, | |||
{"content-type", "cofe/hot; charset=utf-8"} | |||
], "cofe"} | |||
assert expected == | |||
conn(:post, "/") | |||
|> Cache.call(%{query_params: true, ttl: nil}) | |||
|> put_resp_content_type("cofe/hot") | |||
|> send_resp(:ok, "cofe") | |||
|> sent_resp() | |||
end | |||
test "ignore non-successful responses" do | |||
expected = | |||
{418, | |||
[ | |||
{"cache-control", "max-age=0, private, must-revalidate"}, | |||
{"content-type", "tea/iced; charset=utf-8"} | |||
], "🥤"} | |||
assert expected == | |||
conn(:get, "/cofe") | |||
|> Cache.call(%{query_params: true, ttl: nil}) | |||
|> put_resp_content_type("tea/iced") | |||
|> send_resp(:im_a_teapot, "🥤") | |||
|> sent_resp() | |||
end | |||
end |
@@ -175,6 +175,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do | |||
assert json_response(conn, 404) | |||
end | |||
test "it caches a response", %{conn: conn} do | |||
note = insert(:note) | |||
uuid = String.split(note.data["id"], "/") |> List.last() | |||
conn1 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/objects/#{uuid}") | |||
assert json_response(conn1, :ok) | |||
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) | |||
conn2 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/objects/#{uuid}") | |||
assert json_response(conn1, :ok) == json_response(conn2, :ok) | |||
assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"})) | |||
end | |||
test "cached purged after object deletion", %{conn: conn} do | |||
note = insert(:note) | |||
uuid = String.split(note.data["id"], "/") |> List.last() | |||
conn1 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/objects/#{uuid}") | |||
assert json_response(conn1, :ok) | |||
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) | |||
Object.delete(note) | |||
conn2 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/objects/#{uuid}") | |||
assert "Not found" == json_response(conn2, :not_found) | |||
end | |||
end | |||
describe "/object/:uuid/likes" do | |||
@@ -264,6 +307,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do | |||
assert json_response(conn, 404) | |||
end | |||
test "it caches a response", %{conn: conn} do | |||
activity = insert(:note_activity) | |||
uuid = String.split(activity.data["id"], "/") |> List.last() | |||
conn1 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/activities/#{uuid}") | |||
assert json_response(conn1, :ok) | |||
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) | |||
conn2 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/activities/#{uuid}") | |||
assert json_response(conn1, :ok) == json_response(conn2, :ok) | |||
assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"})) | |||
end | |||
test "cached purged after activity deletion", %{conn: conn} do | |||
user = insert(:user) | |||
{:ok, activity} = CommonAPI.post(user, %{"status" => "cofe"}) | |||
uuid = String.split(activity.data["id"], "/") |> List.last() | |||
conn1 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/activities/#{uuid}") | |||
assert json_response(conn1, :ok) | |||
assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) | |||
Activity.delete_by_ap_id(activity.object.data["id"]) | |||
conn2 = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/activities/#{uuid}") | |||
assert "Not found" == json_response(conn2, :not_found) | |||
end | |||
end | |||
describe "/inbox" do | |||