Fix AP C2S pagination Closes #866 and #751 See merge request pleroma/pleroma!2491tags/v2.0.6^2
@@ -18,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
alias Pleroma.Web.ActivityPub.UserView | |||
alias Pleroma.Web.ActivityPub.Utils | |||
alias Pleroma.Web.ActivityPub.Visibility | |||
alias Pleroma.Web.ControllerHelper | |||
alias Pleroma.Web.Federator | |||
require Logger | |||
@@ -200,31 +201,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
end | |||
end | |||
def outbox(conn, %{"nickname" => nickname, "page" => page?} = params) | |||
def outbox( | |||
%{assigns: %{user: for_user}} = conn, | |||
%{"nickname" => nickname, "page" => page?} = params | |||
) | |||
when page? in [true, "true"] do | |||
with %User{} = user <- User.get_cached_by_nickname(nickname), | |||
{:ok, user} <- User.ensure_keys_present(user) do | |||
activities = | |||
if params["max_id"] do | |||
ActivityPub.fetch_user_activities(user, nil, %{ | |||
"max_id" => params["max_id"], | |||
# This is a hack because postgres generates inefficient queries when filtering by | |||
# 'Answer', poll votes will be hidden by the visibility filter in this case anyway | |||
"include_poll_votes" => true, | |||
"limit" => 10 | |||
}) | |||
else | |||
ActivityPub.fetch_user_activities(user, nil, %{ | |||
"limit" => 10, | |||
"include_poll_votes" => true | |||
}) | |||
end | |||
# "include_poll_votes" is a hack because postgres generates inefficient | |||
# queries when filtering by 'Answer', poll votes will be hidden by the | |||
# visibility filter in this case anyway | |||
params = | |||
params | |||
|> Map.drop(["nickname", "page"]) | |||
|> Map.put("include_poll_votes", true) | |||
activities = ActivityPub.fetch_user_activities(user, for_user, params) | |||
conn | |||
|> put_resp_content_type("application/activity+json") | |||
|> put_view(UserView) | |||
|> render("activity_collection_page.json", %{ | |||
activities: activities, | |||
pagination: ControllerHelper.get_pagination_fields(conn, activities), | |||
iri: "#{user.ap_id}/outbox" | |||
}) | |||
end | |||
@@ -318,21 +317,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do | |||
%{"nickname" => nickname, "page" => page?} = params | |||
) | |||
when page? in [true, "true"] do | |||
params = | |||
params | |||
|> Map.drop(["nickname", "page"]) | |||
|> Map.put("blocking_user", user) | |||
|> Map.put("user", user) | |||
activities = | |||
if params["max_id"] do | |||
ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{ | |||
"max_id" => params["max_id"], | |||
"limit" => 10 | |||
}) | |||
else | |||
ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{"limit" => 10}) | |||
end | |||
[user.ap_id | User.following(user)] | |||
|> ActivityPub.fetch_activities(params) | |||
|> Enum.reverse() | |||
conn | |||
|> put_resp_content_type("application/activity+json") | |||
|> put_view(UserView) | |||
|> render("activity_collection_page.json", %{ | |||
activities: activities, | |||
pagination: ControllerHelper.get_pagination_fields(conn, activities), | |||
iri: "#{user.ap_id}/inbox" | |||
}) | |||
end | |||
@@ -216,34 +216,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do | |||
|> Map.merge(Utils.make_json_ld_header()) | |||
end | |||
def render("activity_collection_page.json", %{activities: activities, iri: iri}) do | |||
# this is sorted chronologically, so first activity is the newest (max) | |||
{max_id, min_id, collection} = | |||
if length(activities) > 0 do | |||
{ | |||
Enum.at(activities, 0).id, | |||
Enum.at(Enum.reverse(activities), 0).id, | |||
Enum.map(activities, fn act -> | |||
{:ok, data} = Transmogrifier.prepare_outgoing(act.data) | |||
data | |||
end) | |||
} | |||
else | |||
{ | |||
0, | |||
0, | |||
[] | |||
} | |||
end | |||
def render("activity_collection_page.json", %{ | |||
activities: activities, | |||
iri: iri, | |||
pagination: pagination | |||
}) do | |||
collection = | |||
Enum.map(activities, fn activity -> | |||
{:ok, data} = Transmogrifier.prepare_outgoing(activity.data) | |||
data | |||
end) | |||
%{ | |||
"id" => "#{iri}?max_id=#{max_id}&page=true", | |||
"type" => "OrderedCollectionPage", | |||
"partOf" => iri, | |||
"orderedItems" => collection, | |||
"next" => "#{iri}?max_id=#{min_id}&page=true" | |||
"orderedItems" => collection | |||
} | |||
|> Map.merge(Utils.make_json_ld_header()) | |||
|> Map.merge(pagination) | |||
end | |||
defp maybe_put_total_items(map, false, _total), do: map | |||
@@ -5,7 +5,9 @@ | |||
defmodule Pleroma.Web.ControllerHelper do | |||
use Pleroma.Web, :controller | |||
# As in MastoAPI, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html | |||
alias Pleroma.Pagination | |||
# As in Mastodon API, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html | |||
@falsy_param_values [false, 0, "0", "f", "F", "false", "False", "FALSE", "off", "OFF"] | |||
def truthy_param?(blank_value) when blank_value in [nil, ""], do: nil | |||
def truthy_param?(value), do: value not in @falsy_param_values | |||
@@ -34,38 +36,53 @@ defmodule Pleroma.Web.ControllerHelper do | |||
defp param_to_integer(_, default), do: default | |||
def add_link_headers(conn, activities, extra_params \\ %{}) do | |||
def add_link_headers(conn, activities, extra_params \\ %{}) | |||
def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params), | |||
do: conn | |||
def add_link_headers(conn, activities, extra_params) do | |||
case get_pagination_fields(conn, activities, extra_params) do | |||
%{"next" => next_url, "prev" => prev_url} -> | |||
put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") | |||
_ -> | |||
conn | |||
end | |||
end | |||
def get_pagination_fields(conn, activities, extra_params \\ %{}) do | |||
case List.last(activities) do | |||
%{id: max_id} -> | |||
params = | |||
conn.params | |||
|> Map.drop(Map.keys(conn.path_params)) | |||
|> Map.drop(["since_id", "max_id", "min_id"]) | |||
|> Map.merge(extra_params) | |||
limit = | |||
params | |||
|> Map.get("limit", "20") | |||
|> String.to_integer() | |||
|> Map.drop(Pagination.page_keys() -- ["limit", "order"]) | |||
min_id = | |||
if length(activities) <= limit do | |||
activities | |||
|> List.first() | |||
|> Map.get(:id) | |||
else | |||
activities | |||
|> Enum.at(limit * -1) | |||
|> Map.get(:id) | |||
end | |||
next_url = current_url(conn, Map.merge(params, %{max_id: max_id})) | |||
prev_url = current_url(conn, Map.merge(params, %{min_id: min_id})) | |||
put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") | |||
activities | |||
|> List.first() | |||
|> Map.get(:id) | |||
fields = %{ | |||
"next" => current_url(conn, Map.put(params, :max_id, max_id)), | |||
"prev" => current_url(conn, Map.put(params, :min_id, min_id)) | |||
} | |||
# Generating an `id` without already present pagination keys would | |||
# need a query-restriction with an `q.id >= ^id` or `q.id <= ^id` | |||
# instead of the `q.id > ^min_id` and `q.id < ^max_id`. | |||
# This is because we only have ids present inside of the page, while | |||
# `min_id`, `since_id` and `max_id` requires to know one outside of it. | |||
if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do | |||
Map.put(fields, "id", current_url(conn, conn.params)) | |||
else | |||
fields | |||
end | |||
_ -> | |||
conn | |||
%{} | |||
end | |||
end | |||
@@ -40,10 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do | |||
|> Map.put("muting_user", user) | |||
|> Map.put("user", user) | |||
recipients = [user.ap_id | User.following(user)] | |||
activities = | |||
recipients | |||
[user.ap_id | User.following(user)] | |||
|> ActivityPub.fetch_activities(params) | |||
|> Enum.reverse() | |||
@@ -545,19 +545,13 @@ defmodule Pleroma.Web.Router do | |||
get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe) | |||
end | |||
# Server to Server (S2S) AP interactions | |||
pipeline :activitypub do | |||
plug(:accepts, ["activity+json", "json"]) | |||
plug(Pleroma.Web.Plugs.HTTPSignaturePlug) | |||
plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) | |||
end | |||
scope "/", Pleroma.Web.ActivityPub do | |||
# XXX: not really ostatus | |||
pipe_through(:ostatus) | |||
get("/users/:nickname/outbox", ActivityPubController, :outbox) | |||
plug(:ap_service_actor) | |||
plug(:http_signature) | |||
end | |||
# Client to Server (C2S) AP interactions | |||
pipeline :activitypub_client do | |||
plug(:accepts, ["activity+json", "json"]) | |||
plug(:fetch_session) | |||
@@ -578,6 +572,7 @@ defmodule Pleroma.Web.Router do | |||
get("/api/ap/whoami", ActivityPubController, :whoami) | |||
get("/users/:nickname/inbox", ActivityPubController, :read_inbox) | |||
get("/users/:nickname/outbox", ActivityPubController, :outbox) | |||
post("/users/:nickname/outbox", ActivityPubController, :update_outbox) | |||
post("/api/ap/upload_media", ActivityPubController, :upload_media) | |||
@@ -619,17 +619,63 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do | |||
end | |||
end | |||
describe "/users/:nickname/outbox" do | |||
test "it will not bomb when there is no activity", %{conn: conn} do | |||
describe "GET /users/:nickname/outbox" do | |||
test "it paginates correctly", %{conn: conn} do | |||
user = insert(:user) | |||
conn = assign(conn, :user, user) | |||
outbox_endpoint = user.ap_id <> "/outbox" | |||
_posts = | |||
for i <- 0..25 do | |||
{:ok, activity} = CommonAPI.post(user, %{"status" => "post #{i}"}) | |||
activity | |||
end | |||
result = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get(outbox_endpoint <> "?page=true") | |||
|> json_response(200) | |||
result_ids = Enum.map(result["orderedItems"], fn x -> x["id"] end) | |||
assert length(result["orderedItems"]) == 20 | |||
assert length(result_ids) == 20 | |||
assert result["next"] | |||
assert String.starts_with?(result["next"], outbox_endpoint) | |||
result_next = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get(result["next"]) | |||
|> json_response(200) | |||
result_next_ids = Enum.map(result_next["orderedItems"], fn x -> x["id"] end) | |||
assert length(result_next["orderedItems"]) == 6 | |||
assert length(result_next_ids) == 6 | |||
refute Enum.find(result_next_ids, fn x -> x in result_ids end) | |||
refute Enum.find(result_ids, fn x -> x in result_next_ids end) | |||
assert String.starts_with?(result["id"], outbox_endpoint) | |||
result_next_again = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get(result_next["id"]) | |||
|> json_response(200) | |||
assert result_next == result_next_again | |||
end | |||
test "it returns 200 even if there're no activities", %{conn: conn} do | |||
user = insert(:user) | |||
outbox_endpoint = user.ap_id <> "/outbox" | |||
conn = | |||
conn | |||
|> put_req_header("accept", "application/activity+json") | |||
|> get("/users/#{user.nickname}/outbox") | |||
|> get(outbox_endpoint) | |||
result = json_response(conn, 200) | |||
assert user.ap_id <> "/outbox" == result["id"] | |||
assert outbox_endpoint == result["id"] | |||
end | |||
test "it returns a note activity in a collection", %{conn: conn} do | |||
@@ -158,35 +158,4 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do | |||
assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) | |||
end | |||
end | |||
test "activity collection page aginates correctly" do | |||
user = insert(:user) | |||
posts = | |||
for i <- 0..25 do | |||
{:ok, activity} = CommonAPI.post(user, %{"status" => "post #{i}"}) | |||
activity | |||
end | |||
# outbox sorts chronologically, newest first, with ten per page | |||
posts = Enum.reverse(posts) | |||
%{"next" => next_url} = | |||
UserView.render("activity_collection_page.json", %{ | |||
iri: "#{user.ap_id}/outbox", | |||
activities: Enum.take(posts, 10) | |||
}) | |||
next_id = Enum.at(posts, 9).id | |||
assert next_url =~ next_id | |||
%{"next" => next_url} = | |||
UserView.render("activity_collection_page.json", %{ | |||
iri: "#{user.ap_id}/outbox", | |||
activities: Enum.take(Enum.drop(posts, 10), 10) | |||
}) | |||
next_id = Enum.at(posts, 19).id | |||
assert next_url =~ next_id | |||
end | |||
end |