From 7483679a7b6ff63c9c61c3df3e9e37f2c24012ff Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 31 Jul 2019 15:12:29 +0200 Subject: [PATCH 01/66] StatusView: Return direct conversation id. --- lib/pleroma/conversation/participation.ex | 8 ++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 18 +++++++++++++++++- test/web/mastodon_api/status_view_test.exs | 18 +++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 5883e4183..77b3f61e9 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -65,6 +65,14 @@ defmodule Pleroma.Conversation.Participation do |> Pleroma.Pagination.fetch_paginated(params) end + def for_user_and_conversation(user, conversation) do + from(p in __MODULE__, + where: p.user_id == ^user.id, + where: p.conversation_id == ^conversation.id + ) + |> Repo.one() + end + def for_user_with_last_activity_id(user, params \\ %{}) do for_user(user, params) |> Enum.map(fn participation -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 80df9b2ac..a862554b1 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Activity + alias Pleroma.Conversation + alias Pleroma.Conversation.Participation alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -225,6 +227,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do object.data["url"] || object.data["external_url"] || object.data["id"] end + direct_conversation_id = + with {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, + {_, %User{} = for_user} <- {:for_user, opts[:for]}, + %{data: %{"context" => context}} when is_binary(context) <- activity, + %Conversation{} = conversation <- Conversation.get_for_ap_id(context), + %Participation{id: participation_id} <- + Participation.for_user_and_conversation(for_user, conversation) do + participation_id + else + _e -> + nil + end + %{ id: to_string(activity.id), uri: object.data["id"], @@ -262,7 +277,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do conversation_id: get_context_id(activity), in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, - spoiler_text: %{"text/plain" => summary_plaintext} + spoiler_text: %{"text/plain" => summary_plaintext}, + direct_conversation_id: direct_conversation_id } } end diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 0b167f839..c983b494f 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -23,6 +23,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do :ok end + test "returns the direct conversation id when given the `with_conversation_id` option" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + + status = + StatusView.render("status.json", + activity: activity, + with_direct_conversation_id: true, + for: user + ) + + assert status[:pleroma][:direct_conversation_id] + end + test "returns a temporary ap_id based user for activities missing db users" do user = insert(:user) @@ -133,7 +148,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do conversation_id: convo_id, in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, - spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])} + spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + direct_conversation_id: nil } } From f88560accd801ac88c60344cef93ef00cf136069 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 11:55:41 +0200 Subject: [PATCH 03/66] Conversations: Add recipient list to conversation participation. This enables to address the same group of people every time. --- lib/pleroma/conversation.ex | 11 +++++++ lib/pleroma/conversation/participation.ex | 4 +++ .../conversation/participation_recipient_ship.ex | 34 ++++++++++++++++++++++ lib/pleroma/user.ex | 7 +++++ .../20190205114625_create_thread_mutes.exs | 2 +- ..._conversation_participation_recipient_ships.exs | 13 +++++++++ test/conversation/participation_test.exs | 30 +++++++++++++++++++ 7 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/conversation/participation_recipient_ship.ex create mode 100644 priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index bc97b39ca..fb0dfedca 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Conversation do alias Pleroma.Conversation.Participation + alias Pleroma.Conversation.Participation.RecipientShip alias Pleroma.Repo alias Pleroma.User use Ecto.Schema @@ -39,6 +40,15 @@ defmodule Pleroma.Conversation do Repo.get_by(__MODULE__, ap_id: ap_id) end + def maybe_set_recipients(participation, activity) do + participation = Repo.preload(participation, :recipients) + + if participation.recipients |> Enum.empty?() do + recipients = User.get_all_by_ap_id(activity.recipients) + RecipientShip.create(recipients, participation) + end + end + @doc """ This will 1. Create a conversation if there isn't one already @@ -60,6 +70,7 @@ defmodule Pleroma.Conversation do {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) + maybe_set_recipients(participation, activity) participation end) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 77b3f61e9..121efb671 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Conversation.Participation do use Ecto.Schema alias Pleroma.Conversation + alias Pleroma.Conversation.Participation.RecipientShip alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -17,6 +18,9 @@ defmodule Pleroma.Conversation.Participation do field(:read, :boolean, default: false) field(:last_activity_id, Pleroma.FlakeId, virtual: true) + has_many(:recipient_ships, RecipientShip) + has_many(:recipients, through: [:recipient_ships, :user]) + timestamps() end diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex new file mode 100644 index 000000000..27c0c89cd --- /dev/null +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Conversation.Participation.RecipientShip do + use Ecto.Schema + + alias Pleroma.Conversation.Participation + alias Pleroma.User + alias Pleroma.Repo + + import Ecto.Changeset + + schema "conversation_participation_recipient_ships" do + belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:participation, Participation) + end + + def creation_cng(struct, params) do + struct + |> cast(params, [:user_id, :participation_id]) + |> validate_required([:user_id, :participation_id]) + end + + def create(%User{} = user, participation), do: create([user], participation) + + def create(users, participation) do + Enum.each(users, fn user -> + %__MODULE__{} + |> creation_cng(%{user_id: user.id, participation_id: participation.id}) + |> Repo.insert!() + end) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 6e2fd3af8..a021e77f0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -450,6 +450,13 @@ defmodule Pleroma.User do Repo.get_by(User, ap_id: ap_id) end + def get_all_by_ap_id(ap_ids) do + from(u in __MODULE__, + where: u.ap_id in ^ap_ids + ) + |> Repo.all() + end + # This is mostly an SPC migration fix. This guesses the user nickname by taking the last part # of the ap_id and the domain and tries to get that user def get_by_guessed_nickname(ap_id) do diff --git a/priv/repo/migrations/20190205114625_create_thread_mutes.exs b/priv/repo/migrations/20190205114625_create_thread_mutes.exs index 7e44db121..baaf07253 100644 --- a/priv/repo/migrations/20190205114625_create_thread_mutes.exs +++ b/priv/repo/migrations/20190205114625_create_thread_mutes.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Repo.Migrations.CreateThreadMutes do add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :context, :string end - + create_if_not_exists unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) end end diff --git a/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs new file mode 100644 index 000000000..c6e3469d5 --- /dev/null +++ b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.CreateConversationParticipationRecipientShips do + use Ecto.Migration + + def change do + create_if_not_exists table(:conversation_participation_recipient_ships) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:participation_id, references(:conversation_participations, on_delete: :delete_all)) + end + + create_if_not_exists index(:conversation_participation_recipient_ships, [:user_id]) + create_if_not_exists index(:conversation_participation_recipient_ships, [:participation_id]) + end +end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 2a03e5d67..4a3c397bd 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -8,6 +8,36 @@ defmodule Pleroma.Conversation.ParticipationTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + test "for a new conversation, it sets the recipents of the participation" do + user = insert(:user) + other_user = insert(:user) + third_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + participation = Pleroma.Repo.preload(participation, :recipients) + + assert length(participation.recipients) == 2 + assert user in participation.recipients + assert other_user in participation.recipients + + # Mentioning another user in the same conversation will not add a new recipients. + + {:ok, _activity} = + CommonAPI.post(user, %{ + "in_reply_to_status_id" => activity.id, + "status" => "Hey @#{third_user.nickname}.", + "visibility" => "direct" + }) + + [participation] = Participation.for_user(user) + participation = Pleroma.Repo.preload(participation, :recipients) + + assert length(participation.recipients) == 2 + end + test "it creates a participation for a conversation and a user" do user = insert(:user) conversation = insert(:conversation) From 56b1c3af13c9519e13da688bdbbfdd8d69cda4ac Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 15:05:27 +0200 Subject: [PATCH 04/66] CommonAPI: Extend api with conversation replies. --- lib/pleroma/conversation/participation.ex | 6 ++++++ lib/pleroma/web/common_api/common_api.ex | 20 ++++++++++++++------ lib/pleroma/web/common_api/utils.ex | 27 ++++++++++++++++++++------- test/web/common_api/common_api_test.exs | 30 ++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 121efb671..f1e1a6958 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -93,4 +93,10 @@ defmodule Pleroma.Conversation.Participation do end) |> Enum.filter(& &1.last_activity_id) end + + def get(nil), do: nil + + def get(id) do + Repo.get(__MODULE__, id) + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2db58324b..86e95cd0f 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.ThreadMute @@ -171,21 +172,25 @@ defmodule Pleroma.Web.CommonAPI do end) end - def get_visibility(%{"visibility" => visibility}, in_reply_to) + def get_visibility(_, _, %Participation{}) do + {"direct", "direct"} + end + + def get_visibility(%{"visibility" => visibility}, in_reply_to, _) when visibility in ~w{public unlisted private direct}, do: {visibility, get_replied_to_visibility(in_reply_to)} - def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to) do + def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to, _) do visibility = {:list, String.to_integer(list_id)} {visibility, get_replied_to_visibility(in_reply_to)} end - def get_visibility(_, in_reply_to) when not is_nil(in_reply_to) do + def get_visibility(_, in_reply_to, _) when not is_nil(in_reply_to) do visibility = get_replied_to_visibility(in_reply_to) {visibility, visibility} end - def get_visibility(_, in_reply_to), do: {"public", get_replied_to_visibility(in_reply_to)} + def get_visibility(_, in_reply_to, _), do: {"public", get_replied_to_visibility(in_reply_to)} def get_replied_to_visibility(nil), do: nil @@ -201,7 +206,9 @@ defmodule Pleroma.Web.CommonAPI do with status <- String.trim(status), attachments <- attachments_from_ids(data), in_reply_to <- get_replied_to_activity(data["in_reply_to_status_id"]), - {visibility, in_reply_to_visibility} <- get_visibility(data, in_reply_to), + in_reply_to_conversation <- Participation.get(data["in_reply_to_conversation_id"]), + {visibility, in_reply_to_visibility} <- + get_visibility(data, in_reply_to, in_reply_to_conversation), {_, false} <- {:private_to_public, in_reply_to_visibility == "direct" && visibility != "direct"}, {content_html, mentions, tags} <- @@ -214,7 +221,8 @@ defmodule Pleroma.Web.CommonAPI do mentioned_users <- for({_, mentioned_user} <- mentions, do: mentioned_user.ap_id), addressed_users <- get_addressed_users(mentioned_users, data["to"]), {poll, poll_emoji} <- make_poll_data(data), - {to, cc} <- get_to_and_cc(user, addressed_users, in_reply_to, visibility), + {to, cc} <- + get_to_and_cc(user, addressed_users, in_reply_to, visibility, in_reply_to_conversation), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index d80fffa26..e70ba7d43 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Calendar.Strftime alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -64,9 +65,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do end) end - @spec get_to_and_cc(User.t(), list(String.t()), Activity.t() | nil, String.t()) :: + @spec get_to_and_cc( + User.t(), + list(String.t()), + Activity.t() | nil, + String.t(), + Participation.t() | nil + ) :: {list(String.t()), list(String.t())} - def get_to_and_cc(user, mentioned_users, inReplyTo, "public") do + + def get_to_and_cc(_, _, _, _, %Participation{} = participation) do + participation = Repo.preload(participation, :recipients) + {Enum.map(participation.recipients, & &1.ap_id), []} + end + + def get_to_and_cc(user, mentioned_users, inReplyTo, "public", _) do to = [Pleroma.Constants.as_public() | mentioned_users] cc = [user.follower_address] @@ -77,7 +90,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted") do + def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted", _) do to = [user.follower_address | mentioned_users] cc = [Pleroma.Constants.as_public()] @@ -88,12 +101,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def get_to_and_cc(user, mentioned_users, inReplyTo, "private") do - {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct") + def get_to_and_cc(user, mentioned_users, inReplyTo, "private", _) do + {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct", nil) {[user.follower_address | to], cc} end - def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct") do + def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do if inReplyTo do {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []} else @@ -101,7 +114,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}), do: {mentions, []} + def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}, _), do: {mentions, []} def get_addressed_users(_, to) when is_list(to) do User.get_ap_ids_by_nicknames(to) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 16b3f121d..e2a5bf117 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -12,6 +13,35 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory + test "when replying to a conversation / participation, it only mentions the recipients explicitly declared in the participation" do + har = insert(:user) + jafnhar = insert(:user) + tridi = insert(:user) + + {:ok, activity} = + CommonAPI.post(har, %{ + "status" => "@#{jafnhar.nickname} hey", + "visibility" => "direct" + }) + + assert har.ap_id in activity.recipients + assert jafnhar.ap_id in activity.recipients + + [participation] = Participation.for_user(har) + + {:ok, activity} = + CommonAPI.post(har, %{ + "status" => "I don't really like @#{tridi.nickname}", + "visibility" => "direct", + "in_reply_to_status_id" => activity.id, + "in_reply_to_conversation_id" => participation.id + }) + + assert har.ap_id in activity.recipients + assert jafnhar.ap_id in activity.recipients + refute tridi.ap_id in activity.recipients + end + test "with the safe_dm_mention option set, it does not mention people beyond the initial tags" do har = insert(:user) jafnhar = insert(:user) From eee98aaa738c1aa5f2e4203a61b67648d62965c8 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 2 Aug 2019 19:53:08 +0200 Subject: [PATCH 05/66] Pleroma API: Add endpoint to get conversation statuses. --- lib/pleroma/web/controller_helper.ex | 76 ++++++++++++++++++++++ .../web/mastodon_api/mastodon_api_controller.ex | 68 +------------------ .../web/pleroma_api/pleroma_api_controller.ex | 49 ++++++++++++++ lib/pleroma/web/router.ex | 9 +++ test/web/common_api/common_api_utils_test.exs | 16 ++--- .../pleroma_api/pleroma_api_controller_test.exs | 45 +++++++++++++ 6 files changed, 189 insertions(+), 74 deletions(-) create mode 100644 lib/pleroma/web/pleroma_api/pleroma_api_controller.ex create mode 100644 test/web/pleroma_api/pleroma_api_controller_test.exs diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index 8a753bb4f..eeac9f503 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -33,4 +33,80 @@ defmodule Pleroma.Web.ControllerHelper do end defp param_to_integer(_, default), do: default + + def add_link_headers( + conn, + method, + activities, + param \\ nil, + params \\ %{}, + func3 \\ nil, + func4 \\ nil + ) do + params = + conn.params + |> Map.drop(["since_id", "max_id", "min_id"]) + |> Map.merge(params) + + last = List.last(activities) + + func3 = func3 || (&mastodon_api_url/3) + func4 = func4 || (&mastodon_api_url/4) + + if last do + max_id = last.id + + limit = + params + |> Map.get("limit", "20") + |> String.to_integer() + + min_id = + if length(activities) <= limit do + activities + |> List.first() + |> Map.get(:id) + else + activities + |> Enum.at(limit * -1) + |> Map.get(:id) + end + + {next_url, prev_url} = + if param do + { + func4.( + Pleroma.Web.Endpoint, + method, + param, + Map.merge(params, %{max_id: max_id}) + ), + func4.( + Pleroma.Web.Endpoint, + method, + param, + Map.merge(params, %{min_id: min_id}) + ) + } + else + { + func3.( + Pleroma.Web.Endpoint, + method, + Map.merge(params, %{max_id: max_id}) + ), + func3.( + Pleroma.Web.Endpoint, + method, + Map.merge(params, %{min_id: min_id}) + ) + } + end + + conn + |> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") + else + conn + end + end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 174e93468..0deeab2be 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [json_response: 3] + import Pleroma.Web.ControllerHelper, + only: [json_response: 3, add_link_headers: 5, add_link_headers: 4, add_link_headers: 3] alias Ecto.Changeset alias Pleroma.Activity @@ -342,71 +343,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do json(conn, mastodon_emoji) end - defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do - params = - conn.params - |> Map.drop(["since_id", "max_id", "min_id"]) - |> Map.merge(params) - - last = List.last(activities) - - if last do - max_id = last.id - - limit = - params - |> Map.get("limit", "20") - |> String.to_integer() - - min_id = - if length(activities) <= limit do - activities - |> List.first() - |> Map.get(:id) - else - activities - |> Enum.at(limit * -1) - |> Map.get(:id) - end - - {next_url, prev_url} = - if param do - { - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{max_id: max_id}) - ), - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{min_id: min_id}) - ) - } - else - { - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{max_id: max_id}) - ), - mastodon_api_url( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{min_id: min_id}) - ) - } - end - - conn - |> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") - else - conn - end - end - def home_timeline(%{assigns: %{user: user}} = conn, params) do params = params diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex new file mode 100644 index 000000000..b677892ed --- /dev/null +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] + + alias Pleroma.Conversation.Participation + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Repo + + def conversation_statuses( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id} = params + ) do + params = + params + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + + participation = + participation_id + |> Participation.get() + |> Repo.preload(:conversation) + + if user.id == participation.user_id do + activities = + participation.conversation.ap_id + |> ActivityPub.fetch_activities_for_context(params) + |> Enum.reverse() + + conn + |> add_link_headers( + :conversation_statuses, + activities, + participation_id, + params, + nil, + &pleroma_api_url/4 + ) + |> put_view(StatusView) + |> render("index.json", %{activities: activities, for: user, as: :activity}) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 0689d69fb..40298538a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -257,6 +257,15 @@ defmodule Pleroma.Web.Router do end end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:authenticated_api) + + scope [] do + pipe_through(:oauth_write) + get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + end + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index af320f31f..7510c8def 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -239,7 +239,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "public") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "public", nil) assert length(to) == 2 assert length(cc) == 1 @@ -256,7 +256,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "public") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "public", nil) assert length(to) == 3 assert length(cc) == 1 @@ -272,7 +272,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "unlisted") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "unlisted", nil) assert length(to) == 2 assert length(cc) == 1 @@ -289,7 +289,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "unlisted") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "unlisted", nil) assert length(to) == 3 assert length(cc) == 1 @@ -305,7 +305,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "private") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "private", nil) assert length(to) == 2 assert length(cc) == 0 @@ -321,7 +321,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "private") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "private", nil) assert length(to) == 3 assert length(cc) == 0 @@ -336,7 +336,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do mentioned_user = insert(:user) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "direct") + {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "direct", nil) assert length(to) == 1 assert length(cc) == 0 @@ -351,7 +351,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "uguu"}) mentions = [mentioned_user.ap_id] - {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "direct") + {to, cc} = Utils.get_to_and_cc(user, mentions, activity, "direct", nil) assert length(to) == 2 assert length(cc) == 0 diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs new file mode 100644 index 000000000..43104e36e --- /dev/null +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Conversation.Participation + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + test "/api/v1/pleroma/conversations/:id/statuses", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + third_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{third_user.nickname}!", "visibility" => "direct"}) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}!", "visibility" => "direct"}) + + [participation] = Participation.for_user(other_user) + + {:ok, activity_two} = + CommonAPI.post(other_user, %{ + "status" => "Hi!", + "in_reply_to_status_id" => activity.id, + "in_reply_to_conversation_id" => participation.id + }) + + result = + conn + |> assign(:user, other_user) + |> get("/api/v1/pleroma/conversations/#{participation.id}/statuses") + |> json_response(200) + + assert length(result) == 2 + + id_one = activity.id + id_two = activity_two.id + assert [%{"id" => ^id_one}, %{"id" => ^id_two}] = result + end +end From 3af6d14da769aa5adfdd6360b43c691fd8c8eed5 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:09:19 +0200 Subject: [PATCH 06/66] Pleroma Conversations API: Add a way to set recipients. --- lib/pleroma/conversation/participation.ex | 20 +++++++++++ .../web/mastodon_api/views/conversation_view.ex | 13 +++++-- .../web/pleroma_api/pleroma_api_controller.ex | 17 +++++++++ lib/pleroma/web/router.ex | 1 + test/web/mastodon_api/conversation_view_test.exs | 40 ++++++++++++++++++++++ .../pleroma_api/pleroma_api_controller_test.exs | 31 +++++++++++++++++ 6 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 test/web/mastodon_api/conversation_view_test.exs diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index f1e1a6958..acdee5517 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -99,4 +99,24 @@ defmodule Pleroma.Conversation.Participation do def get(id) do Repo.get(__MODULE__, id) end + + def set_recipients(participation, user_ids) do + Repo.transaction(fn -> + query = + from(r in RecipientShip, + where: r.participation_id == ^participation.id + ) + + Repo.delete_all(query) + + users = + from(u in User, + where: u.id in ^user_ids + ) + |> Repo.all() + + RecipientShip.create(users, participation) + :ok + end) + end end diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 38bdec737..5adaecdb0 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participation.json", %{participation: participation, user: user}) do - participation = Repo.preload(participation, conversation: :users) + participation = Repo.preload(participation, conversation: :users, recipients: []) last_activity_id = with nil <- participation.last_activity_id do @@ -37,11 +37,20 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do as: :user }) + recipients = + AccountView.render("accounts.json", %{ + users: participation.recipients, + as: :user + }) + %{ id: participation.id |> to_string(), accounts: accounts, unread: !participation.read, - last_status: last_status + last_status: last_status, + pleroma: %{ + recipients: recipients + } } end end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index b677892ed..759d8aef0 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do alias Pleroma.Conversation.Participation alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Repo def conversation_statuses( @@ -46,4 +47,20 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do |> render("index.json", %{activities: activities, for: user, as: :activity}) end end + + def update_conversation( + %{assigns: %{user: user}} = conn, + %{"id" => participation_id, "recipients" => recipients} + ) do + participation = + participation_id + |> Participation.get() + + with true <- user.id == participation.user_id, + {:ok, _} <- Participation.set_recipients(participation, recipients) do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, user: user}) + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 40298538a..6cdef7e2f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -263,6 +263,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + patch("/conversations/:id", PleromaAPIController, :update_conversation) end end diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs new file mode 100644 index 000000000..2a4b41fa4 --- /dev/null +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do + use Pleroma.DataCase + + alias Pleroma.Web.CommonAPI + alias Pleroma.Conversation.Participation + alias Pleroma.Web.MastodonAPI.ConversationView + + import Pleroma.Factory + + test "represents a Mastodon Conversation entity" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "hey @#{other_user.nickname}", "visibility" => "direct"}) + + [participation] = Participation.for_user_with_last_activity_id(user) + + assert participation + + conversation = + ConversationView.render("participation.json", %{participation: participation, user: user}) + + assert conversation.id == participation.id |> to_string() + assert conversation.last_status.id == activity.id + + assert [account] = conversation.accounts + assert account.id == other_user.id + + assert recipients = conversation.pleroma.recipients + recipient_ids = recipients |> Enum.map(& &1.id) + + assert user.id in recipient_ids + assert other_user.id in recipient_ids + end +end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 43104e36e..7989defe0 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + alias Pleroma.Repo import Pleroma.Factory @@ -42,4 +43,34 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do id_two = activity_two.id assert [%{"id" => ^id_one}, %{"id" => ^id_two}] = result end + + test "PATCH /api/v1/pleroma/conversations/:id", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = CommonAPI.post(user, %{"status" => "Hi", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + participation = Repo.preload(participation, :recipients) + + assert [user] == participation.recipients + assert other_user not in participation.recipients + + result = + conn + |> assign(:user, user) + |> patch("/api/v1/pleroma/conversations/#{participation.id}", %{ + "recipients" => [user.id, other_user.id] + }) + |> json_response(200) + + assert result["id"] == participation.id |> to_string + + assert recipients = result["pleroma"]["recipients"] + recipient_ids = Enum.map(recipients, & &1["id"]) + + assert user.id in recipient_ids + assert other_user.id in recipient_ids + end end From b64b6fee2a78fbfbc557b89550128494ca7d2894 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:33:22 +0200 Subject: [PATCH 07/66] CommonAPI: Replies to conversations also get the correct context id. --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- test/web/common_api/common_api_test.exs | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 86e95cd0f..72da46263 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -223,7 +223,7 @@ defmodule Pleroma.Web.CommonAPI do {poll, poll_emoji} <- make_poll_data(data), {to, cc} <- get_to_and_cc(user, addressed_users, in_reply_to, visibility, in_reply_to_conversation), - context <- make_context(in_reply_to), + context <- make_context(in_reply_to, in_reply_to_conversation), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), full_payload <- String.trim(status <> cw), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index e70ba7d43..425b6d656 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -244,8 +244,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do defp maybe_add_nsfw_tag(data, _), do: data - def make_context(%Activity{data: %{"context" => context}}), do: context - def make_context(_), do: Utils.generate_context_id() + def make_context(_, %Participation{} = participation) do + Repo.preload(participation, :conversation).conversation.ap_id + end + + def make_context(%Activity{data: %{"context" => context}}, _), do: context + def make_context(_, _), do: Utils.generate_context_id() def maybe_add_attachments(parsed, _attachments, true = _no_links), do: parsed diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e2a5bf117..454523349 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -9,10 +9,25 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI import Pleroma.Factory + test "when replying to a conversation / participation, it will set the correct context id even if no explicit reply_to is given" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + {:ok, convo_reply} = + CommonAPI.post(user, %{"status" => ".", "in_reply_to_conversation_id" => participation.id}) + + assert Visibility.is_direct?(convo_reply) + + assert activity.data["context"] == convo_reply.data["context"] + end + test "when replying to a conversation / participation, it only mentions the recipients explicitly declared in the participation" do har = insert(:user) jafnhar = insert(:user) From ddabe7784b47939120a838b9c65381fd2262c161 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 15:58:14 +0200 Subject: [PATCH 08/66] Pleroma Conversations: Document differences. --- docs/api/differences_in_mastoapi_responses.md | 8 ++++++++ docs/api/pleroma_api.md | 29 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 1907d70c8..79ca531b8 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -59,12 +59,19 @@ Has these additional fields under the `pleroma` object: - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API +## Conversations + +Has an additional field under the `pleroma` object: + +- `recipients`: The list of the recipients of this Conversation. These will be addressed when replying to this conversation. + ## Account Search Behavior has changed: - `/api/v1/accounts/search`: Does not require authentication + ## Notifications Has these additional fields under the `pleroma` object: @@ -79,6 +86,7 @@ Additional parameters can be added to the JSON body/Form data: - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. +- `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. ## PATCH `/api/v1/update_credentials` diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 5698e88ac..4323e59f0 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -319,3 +319,32 @@ See [Admin-API](Admin-API.md) "healthy": true # Instance state } ``` + +# Pleroma Conversations + +Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints: + +1. Pleroma Conversations never add or remove recipients, unless explicitly changed by the user. +2. Pleroma Conversations statuses can be requested by Conversation id. +3. Pleroma Conversations can be replied to. + +Conversations have the additional field "recipients" under the "pleroma" key. This holds a list of all the accounts that will receive a message in this conversation. + +The status posting endpoint takes an additional parameter, `in_reply_to_conversation_id`, which, when set, will set the visiblity to direct and address only the people who are the recipients of that Conversation. + + +## `GET /api/v1/pleroma/conversations/:id/statuses` +### Timeline for a given conversation +* Method `GET` +* Authentication: required +* Params: Like other timelines +* Response: JSON, statuses (200 - healthy, 503 unhealthy). + + +## `PATCH /api/v1/pleroma/conversations/:id` +### Update a conversation. Used to change the set of recipients. +* Method `PATCH` +* Authentication: required +* Params: + * `recipients`: A list of ids of users that should receive posts to this conversation. +* Response: JSON, statuses (200 - healthy, 503 unhealthy) From d6fe220e32d581220cc33f4f44d6517a401eabbf Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 5 Aug 2019 16:11:23 +0200 Subject: [PATCH 09/66] Linting. --- lib/pleroma/conversation/participation_recipient_ship.ex | 2 +- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 6 +++--- test/web/mastodon_api/conversation_view_test.exs | 2 +- test/web/pleroma_api/pleroma_api_controller_test.exs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex index 27c0c89cd..932cbd04c 100644 --- a/lib/pleroma/conversation/participation_recipient_ship.ex +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Conversation.Participation.RecipientShip do use Ecto.Schema alias Pleroma.Conversation.Participation - alias Pleroma.User alias Pleroma.Repo + alias Pleroma.User import Ecto.Changeset diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 759d8aef0..018564452 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Conversation.Participation - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Repo + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.ConversationView + alias Pleroma.Web.MastodonAPI.StatusView def conversation_statuses( %{assigns: %{user: user}} = conn, diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index 2a4b41fa4..e32cde5a8 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -5,8 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do use Pleroma.DataCase - alias Pleroma.Web.CommonAPI alias Pleroma.Conversation.Participation + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ConversationView import Pleroma.Factory diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 7989defe0..7c75fb229 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Conversation.Participation - alias Pleroma.Web.CommonAPI alias Pleroma.Repo + alias Pleroma.Web.CommonAPI import Pleroma.Factory From a49c92f6ae2dc68a80345cff4793820a75835eb1 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 6 Aug 2019 14:51:17 +0200 Subject: [PATCH 10/66] Participation: Setting recipients will always add the owner. --- docs/api/pleroma_api.md | 2 +- lib/pleroma/conversation/participation.ex | 6 ++++++ test/conversation/participation_test.exs | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 4323e59f0..590f2a3fb 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -346,5 +346,5 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Method `PATCH` * Authentication: required * Params: - * `recipients`: A list of ids of users that should receive posts to this conversation. + * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index acdee5517..d17b6f7c5 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -101,6 +101,10 @@ defmodule Pleroma.Conversation.Participation do end def set_recipients(participation, user_ids) do + user_ids = + [participation.user_id | user_ids] + |> Enum.uniq() + Repo.transaction(fn -> query = from(r in RecipientShip, @@ -118,5 +122,7 @@ defmodule Pleroma.Conversation.Participation do RecipientShip.create(users, participation) :ok end) + + {:ok, Repo.preload(participation, :recipients, force: true)} end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 4a3c397bd..7958e8e89 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -132,4 +132,23 @@ defmodule Pleroma.Conversation.ParticipationTest do [] = Participation.for_user_with_last_activity_id(user) end + + test "it sets recipients, always keeping the owner of the participation even when not explicitly set" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + [participation] = Participation.for_user_with_last_activity_id(user) + + participation = Repo.preload(participation, :recipients) + + assert participation.recipients |> length() == 1 + assert user in participation.recipients + + {:ok, participation} = Participation.set_recipients(participation, [other_user.id]) + + assert participation.recipients |> length() == 2 + assert user in participation.recipients + assert other_user in participation.recipients + end end From e4a01d253ef7ab09d028198e5e39b9aba357486c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 6 Aug 2019 15:06:19 +0200 Subject: [PATCH 11/66] Conversation: Rename function to better express what it does. --- lib/pleroma/conversation.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index fb0dfedca..be5821ad7 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Conversation do Repo.get_by(__MODULE__, ap_id: ap_id) end - def maybe_set_recipients(participation, activity) do + def maybe_create_recipientships(participation, activity) do participation = Repo.preload(participation, :recipients) if participation.recipients |> Enum.empty?() do @@ -70,7 +70,7 @@ defmodule Pleroma.Conversation do {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) - maybe_set_recipients(participation, activity) + maybe_create_recipientships(participation, activity) participation end) From dfae61c25c7ee2bb8add38b2cbaa8391f03c9550 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 9 Aug 2019 23:05:28 +0300 Subject: [PATCH 12/66] Fix deactivated user deletion --- lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++++----------- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++--- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/admin_api/admin_api_controller.ex | 4 +-- lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 8 +++++ 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index a3f8bc945..f33d01429 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user) + User.perform(:delete, user, nil) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d18f099e..14057a0e4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,13 +1029,26 @@ defmodule Pleroma.User do |> update_and_set_cache() end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + @spec delete(User.t()) :: :ok - def delete(%User{} = user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + def delete(%User{} = user, actor \\ nil), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user) do - {:ok, _user} = ActivityPub.delete(user) + def perform(:delete, %User{} = user, actor) do + {:ok, _user} = ActivityPub.delete(user, actor: actor) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1057,19 +1070,6 @@ defmodule Pleroma.User do Repo.delete(user) end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a279a7df..8f669acb9 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -403,11 +403,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do + def delete(data, opts \\ %{actor: nil, local: true}) + + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => ap_id, + "actor" => opts[:actor] || ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, {:ok, activity} <- insert(data, true, true), @@ -416,7 +418,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -428,7 +430,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, local, false), + {:ok, activity} <- insert(data, opts[:local], false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d8..b34ef73c0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, false) do + {:ok, activity} <- ActivityPub.delete(object, local: false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2d3d0adc4..63c9a7d7f 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(conn, %{"nickname" => nickname}) do + def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete() + |> User.delete(admin.ap_id) conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index b2f9f3946..ac2dc115c 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, false) do + {:ok, delete} <- ActivityPub.delete(object, local: false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index 8440d456d..e2da8d84b 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -998,6 +998,14 @@ defmodule Pleroma.UserTest do refute Activity.get_by_id(activity.id) end + test "it deletes deactivated user" do + admin = insert(:user, %{info: %{is_admin: true}}) + {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() + + assert {:ok, _} = User.delete(user, admin.ap_id) + refute User.get_by_id(user.id) + end + test "it deletes a user, all follow relationships and all activities", %{user: user} do follower = insert(:user) {:ok, follower} = User.follow(follower, user) From 23c46f7e72701b773d87b825526450e5f4ec6322 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 12:51:08 +0200 Subject: [PATCH 13/66] Conversations: Use 'recipients' for accounts in conversation view. According to gargron, this is the intended usage. --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 15 +++------------ test/web/mastodon_api/conversation_view_test.exs | 6 ------ test/web/pleroma_api/pleroma_api_controller_test.exs | 8 ++++---- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 5adaecdb0..4a81f0248 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participation.json", %{participation: participation, user: user}) do - participation = Repo.preload(participation, conversation: :users, recipients: []) + participation = Repo.preload(participation, conversation: [], recipients: []) last_activity_id = with nil <- participation.last_activity_id do @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do # Conversations return all users except the current user. users = - participation.conversation.users + participation.recipients |> Enum.reject(&(&1.id == user.id)) accounts = @@ -37,20 +37,11 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do as: :user }) - recipients = - AccountView.render("accounts.json", %{ - users: participation.recipients, - as: :user - }) - %{ id: participation.id |> to_string(), accounts: accounts, unread: !participation.read, - last_status: last_status, - pleroma: %{ - recipients: recipients - } + last_status: last_status } end end diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index e32cde5a8..27f668d9f 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -30,11 +30,5 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do assert [account] = conversation.accounts assert account.id == other_user.id - - assert recipients = conversation.pleroma.recipients - recipient_ids = recipients |> Enum.map(& &1.id) - - assert user.id in recipient_ids - assert other_user.id in recipient_ids end end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 7c75fb229..56bc1572c 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -67,10 +67,10 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do assert result["id"] == participation.id |> to_string - assert recipients = result["pleroma"]["recipients"] - recipient_ids = Enum.map(recipients, & &1["id"]) + [participation] = Participation.for_user(user) + participation = Repo.preload(participation, :recipients) - assert user.id in recipient_ids - assert other_user.id in recipient_ids + assert user in participation.recipients + assert other_user in participation.recipients end end From 60231ec7bd0af993dc19f69a57b261b3b4167636 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 13:58:04 +0200 Subject: [PATCH 14/66] Conversation: Add endpoint to get a conversation by id. --- docs/api/pleroma_api.md | 6 ++++++ lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 9 +++++++++ lib/pleroma/web/router.ex | 1 + test/web/pleroma_api/pleroma_api_controller_test.exs | 18 ++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 590f2a3fb..b134b31a8 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -340,6 +340,12 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: Like other timelines * Response: JSON, statuses (200 - healthy, 503 unhealthy). +## `GET /api/v1/pleroma/conversations/:id` +### The conversation with the given ID. +* Method `GET` +* Authentication: required +* Params: None +* Response: JSON, statuses (200 - healthy, 503 unhealthy). ## `PATCH /api/v1/pleroma/conversations/:id` ### Update a conversation. Used to change the set of recipients. diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 018564452..3175a99b1 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -13,6 +13,15 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView + def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do + with %Participation{} = participation <- Participation.get(participation_id), + true <- user.id == participation.user_id do + conn + |> put_view(ConversationView) + |> render("participation.json", %{participation: participation, user: user}) + end + end + def conversation_statuses( %{assigns: %{user: user}} = conn, %{"id" => participation_id} = params diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c835f06b4..f0b6a02e9 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -265,6 +265,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) + get("/conversations/:id", PleromaAPIController, :conversation) patch("/conversations/:id", PleromaAPIController, :update_conversation) end end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 56bc1572c..ed6b79727 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -11,6 +11,24 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do import Pleroma.Factory + test "/api/v1/pleroma/conversations/:id", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}!", "visibility" => "direct"}) + + [participation] = Participation.for_user(other_user) + + result = + conn + |> assign(:user, other_user) + |> get("/api/v1/pleroma/conversations/#{participation.id}") + |> json_response(200) + + assert result["id"] == participation.id |> to_string() + end + test "/api/v1/pleroma/conversations/:id/statuses", %{conn: conn} do user = insert(:user) other_user = insert(:user) From 511ccea5aa36b4b0098e49b409b335b0ce8f042e Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 14:23:06 +0200 Subject: [PATCH 15/66] ConversationView: Align parameter names with other views. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 2 +- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 4 ++-- lib/pleroma/web/streamer.ex | 2 +- test/web/mastodon_api/conversation_view_test.exs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0deeab2be..eb2351eb7 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1743,7 +1743,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do conversations = Enum.map(participations, fn participation -> - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) end) conn @@ -1756,7 +1756,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do Repo.get_by(Participation, id: participation_id, user_id: user.id), {:ok, participation} <- Participation.mark_as_read(participation) do participation_view = - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) conn |> json(participation_view) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 4a81f0248..40acc07b3 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView - def render("participation.json", %{participation: participation, user: user}) do + def render("participation.json", %{participation: participation, for: user}) do participation = Repo.preload(participation, conversation: [], recipients: []) last_activity_id = diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 3175a99b1..b5c3d2728 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do true <- user.id == participation.user_id do conn |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, user: user}) + |> render("participation.json", %{participation: participation, for: user}) end end @@ -69,7 +69,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do {:ok, _} <- Participation.set_recipients(participation, recipients) do conn |> put_view(ConversationView) - |> render("participation.json", %{participation: participation, user: user}) + |> render("participation.json", %{participation: participation, for: user}) end end end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 9ee331030..a0bb10895 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -209,7 +209,7 @@ defmodule Pleroma.Web.Streamer do payload: Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ participation: participation, - user: participation.user + for: participation.user }) |> Jason.encode!() } diff --git a/test/web/mastodon_api/conversation_view_test.exs b/test/web/mastodon_api/conversation_view_test.exs index 27f668d9f..a2a880705 100644 --- a/test/web/mastodon_api/conversation_view_test.exs +++ b/test/web/mastodon_api/conversation_view_test.exs @@ -23,7 +23,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do assert participation conversation = - ConversationView.render("participation.json", %{participation: participation, user: user}) + ConversationView.render("participation.json", %{participation: participation, for: user}) assert conversation.id == participation.id |> to_string() assert conversation.last_status.id == activity.id From 2674db14a2ee29e98265c0c0b1db412835b6bbed Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 12 Aug 2019 14:26:18 +0200 Subject: [PATCH 16/66] Modify Changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 069974e44..f8c90a73b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Report email not being sent to admins when the reporter is a remote user ### Added +- Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data. - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - MRF: Support for excluding specific domains from Transparency. - MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`) From e0ac5c7a66664c897e1b3af9a55e0b73f32fa034 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:26:35 +0700 Subject: [PATCH 17/66] Add custom profile fields --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 31 +++++++++++++++++ .../web/mastodon_api/mastodon_api_controller.ex | 11 ++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 9 +++-- .../update_credentials_test.exs | 39 ++++++++++++++++++++++ 6 files changed, 87 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 758661120..109cf6516 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,6 +255,7 @@ config :pleroma, :instance, dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, + max_account_fields: 4, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index 20311db54..ca5da7db1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,6 +132,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 22eb9a182..fa57052fb 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -49,6 +49,7 @@ defmodule Pleroma.User.Info do field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) + field(:fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -286,10 +287,32 @@ defmodule Pleroma.User.Info do :background, :show_role, :skip_thread_containment, + :fields, :pleroma_settings_store ]) + |> validate_fields() end + def validate_fields(changeset) do + limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + + changeset + |> validate_length(:fields, max: limit) + |> validate_change(:fields, fn :fields, fields -> + if Enum.all?(fields, &valid_field?/1) do + [] + else + [fields: "invalid"] + end + end) + end + + defp valid_field?(%{"name" => name, "value" => value}) do + is_binary(name) && is_binary(value) + end + + defp valid_field?(_), do: false + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) @@ -384,6 +407,14 @@ defmodule Pleroma.User.Info do cast(info, params, [:muted_reblogs]) end + def fields(%{source_data: %{"attachment" => attachment}}) do + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + end + + def fields(%{fields: fields}), do: fields + def follow_information_update(info, params) do info |> cast(params, [ diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 7ce2b5b06..e79a02caa 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -156,6 +156,17 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) end) |> add_if_present(params, "default_scope", :default_scope) + |> add_if_present(params, "fields", :fields, fn fields -> + fields = + Enum.map(fields, fn field -> + %{ + "name" => Formatter.html_escape(field["name"], "text/plain"), + "value" => Formatter.html_escape(field["value"], "text/plain") + } + end) + + {:ok, fields} + end) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 72c092f25..d2f3986ff 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,10 +93,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) + fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) @@ -119,11 +117,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header: header, header_static: header, emojis: emojis, - fields: fields, + fields: fields_html, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, + fields: fields, pleroma: %{} }, diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 71d0c8af8..a3eadde16 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -300,5 +300,44 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do assert user["display_name"] == name assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"] end + + test "update fields", %{conn: conn} do + user = insert(:user) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + account = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(200) + + assert account["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + assert account["source"]["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + Pleroma.Config.put([:instance, :max_account_fields], 1) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + + assert %{"error" => "Invalid request"} == json_response(conn, 403) + end end end From d6094b405d1a744eaa1c17752b3088dfee16c8d2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:48:15 +0700 Subject: [PATCH 18/66] Fix tests --- test/web/mastodon_api/account_view_test.exs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a26f514a5..1d8b28339 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -67,7 +67,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: "valid html", sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: "https://example.com/images/asuka_hospital.png", @@ -134,7 +135,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, @@ -304,7 +306,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, From 069951722f9b674a759e0b8683aff9c03019467b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 14:34:52 +0700 Subject: [PATCH 19/66] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 358287096..a54b04658 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id +- Mastodon API: Add support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From a22f540fc42dd941631e94fe931d1f655b2904a1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:33:18 +0700 Subject: [PATCH 20/66] Add custom fields to TwitterAPI.UserView --- lib/pleroma/user/info.ex | 2 ++ lib/pleroma/web/twitter_api/views/user_view.ex | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fa57052fb..98b894223 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -407,6 +407,8 @@ defmodule Pleroma.User.Info do cast(info, params, [:muted_reblogs]) end + # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. + # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 8d8892068..3681773be 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,12 +74,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. - # For example: [{"name": "Pronoun", "value": "she/her"}, …] - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) data = %{ From 88598c9bafcdcf89b0f1fb00d0785c77b583cd65 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:35:34 +0700 Subject: [PATCH 21/66] Add profile custom fields to ActivityPub.UserView --- lib/pleroma/web/activity_pub/views/user_view.ex | 6 ++++++ test/web/activity_pub/views/user_view_test.exs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 06c9e1c71..7b4bc998b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -80,6 +80,11 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Transmogrifier.add_emoji_tags() |> Map.get("tag", []) + fields = + user.info + |> User.Info.fields() + |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + %{ "id" => user.ap_id, "type" => "Person", @@ -98,6 +103,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "publicKeyPem" => public_key }, "endpoints" => endpoints, + "attachment" => fields, "tag" => (user.info.source_data["tag"] || []) ++ user_tags } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 86254117f..48a522c6c 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -22,6 +22,22 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY") end + test "Renders profile fields" do + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "website", "value" => "cofe.my"} + ] + + user = insert(:user, info: %{fields: fields}) + + assert %{ + "attachment" => [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + ] + } = UserView.render("user.json", %{user: user}) + end + test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) {:ok, user} = User.ensure_keys_present(user) From 8ab87ce40b0b8a5be7cd3576eddbacc279d8d3a1 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 26 Jul 2019 12:08:46 +0000 Subject: [PATCH 22/66] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54b04658..483c88073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id -- Mastodon API: Add support for the user profile custom fields +- Mastodon API: Improve support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From 5178f960c3f5a35e2071bd5463b537cadc9a53af Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:01:15 +0700 Subject: [PATCH 23/66] Support user attachment update in Transmogrifier --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 + test/fixtures/mastodon-update.json | 36 ++++++++++---- .../tesla_mock/admin@mastdon.example.org.json | 55 +++++++++++++++++++++- test/web/activity_pub/transmogrifier_test.exs | 36 ++++++++++++++ 4 files changed, 120 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0fcc81bf3..225c34875 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,11 +598,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false + attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) |> Map.put(:info, %{banner: banner, locked: locked}) + |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) actor |> User.upgrade_changeset(update_data) diff --git a/test/fixtures/mastodon-update.json b/test/fixtures/mastodon-update.json index f6713fea5..dbf8b6dff 100644 --- a/test/fixtures/mastodon-update.json +++ b/test/fixtures/mastodon-update.json @@ -1,10 +1,10 @@ -{ - "type": "Update", - "object": { - "url": "http://mastodon.example.org/@gargron", - "type": "Person", - "summary": "

Some bio

", - "publicKey": { +{ + "type": "Update", + "object": { + "url": "http://mastodon.example.org/@gargron", + "type": "Person", + "summary": "

Some bio

", + "publicKey": { "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0gs3VnQf6am3R+CeBV4H\nlfI1HZTNRIBHgvFszRZkCERbRgEWMu+P+I6/7GJC5H5jhVQ60z4MmXcyHOGmYMK/\n5XyuHQz7V2Ssu1AxLfRN5Biq1ayb0+DT/E7QxNXDJPqSTnstZ6C7zKH/uAETqg3l\nBonjCQWyds+IYbQYxf5Sp3yhvQ80lMwHML3DaNCMlXWLoOnrOX5/yK5+dedesg2\n/HIvGk+HEt36vm6hoH7bwPuEkgA++ACqwjXRe5Mta7i3eilHxFaF8XIrJFARV0t\nqOu4GID/jG6oA+swIWndGrtR2QRJIt9QIBFfK3HG5M0koZbY1eTqwNFRHFL3xaD\nUQIDAQAB\n-----END PUBLIC KEY-----\n", "owner": "http://mastodon.example.org/users/gargron", "id": "http://mastodon.example.org/users/gargron#main-key" @@ -20,7 +20,27 @@ "endpoints": { "sharedInbox": "http://mastodon.example.org/inbox" }, - "icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"} + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "updated" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "updated" + } + ], + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } }, "id": "http://mastodon.example.org/users/gargron#updates/1519563538", "actor": "http://mastodon.example.org/users/gargron", diff --git a/test/fixtures/tesla_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json index c297e4349..8159dc20a 100644 --- a/test/fixtures/tesla_mock/admin@mastdon.example.org.json +++ b/test/fixtures/tesla_mock/admin@mastdon.example.org.json @@ -1 +1,54 @@ -{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","movedTo":"as:movedTo","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji"}],"id":"http://mastodon.example.org/users/admin","type":"Person","following":"http://mastodon.example.org/users/admin/following","followers":"http://mastodon.example.org/users/admin/followers","inbox":"http://mastodon.example.org/users/admin/inbox","outbox":"http://mastodon.example.org/users/admin/outbox","preferredUsername":"admin","name":null,"summary":"\u003cp\u003e\u003c/p\u003e","url":"http://mastodon.example.org/@admin","manuallyApprovesFollowers":false,"publicKey":{"id":"http://mastodon.example.org/users/admin#main-key","owner":"http://mastodon.example.org/users/admin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"http://mastodon.example.org/inbox"},"icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"}} +{ + "@context": ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "movedTo": "as:movedTo", + "Hashtag": "as:Hashtag", + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji" + }], + "id": "http://mastodon.example.org/users/admin", + "type": "Person", + "following": "http://mastodon.example.org/users/admin/following", + "followers": "http://mastodon.example.org/users/admin/followers", + "inbox": "http://mastodon.example.org/users/admin/inbox", + "outbox": "http://mastodon.example.org/users/admin/outbox", + "preferredUsername": "admin", + "name": null, + "summary": "\u003cp\u003e\u003c/p\u003e", + "url": "http://mastodon.example.org/@admin", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "http://mastodon.example.org/users/admin#main-key", + "owner": "http://mastodon.example.org/users/admin", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "bar" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "bar1" + } + ], + "endpoints": { + "sharedInbox": "http://mastodon.example.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 060b91e29..05ec09ec1 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -509,6 +509,42 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert user.bio == "

Some bio

" end + test "it works with custom profile fields" do + {:ok, activity} = + "test/fixtures/mastodon-post-activity.json" + |> File.read!() + |> Poison.decode!() + |> Transmogrifier.handle_incoming() + + user = User.get_cached_by_ap_id(activity.actor) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + ] + + update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() + + object = + update_data["object"] + |> Map.put("actor", user.ap_id) + |> Map.put("id", user.ap_id) + + update_data = + update_data + |> Map.put("actor", user.ap_id) + |> Map.put("object", object) + + {:ok, _update_activity} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + ] + end + test "it works for incoming update activities which lock the account" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From 7d6f8a7fd75e5de4e0c9ce208ac9276dcbe044f5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:17:09 +0700 Subject: [PATCH 24/66] Linkify custom fields values in ActivityPub.UserViewx --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + test/web/activity_pub/views/user_view_test.exs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 7b4bc998b..b2a22478d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -84,6 +84,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do user.info |> User.Info.fields() |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 48a522c6c..a2aa52381 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -33,7 +33,11 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert %{ "attachment" => [ %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + %{ + "name" => "website", + "type" => "PropertyValue", + "value" => "cofe.my" + } ] } = UserView.render("user.json", %{user: user}) end From db3c05f6b4c226733633a409cb1f1a290db4c48b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 30 Jul 2019 17:22:52 +0700 Subject: [PATCH 25/66] Add configurable account field value length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 7 ++++- .../update_credentials_test.exs | 31 +++++++++++++++++----- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 109cf6516..21f4861f2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_value_length: 255, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index ca5da7db1..fbb506093 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 98b894223..9e4d381f8 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,7 +308,12 @@ defmodule Pleroma.User.Info do end defp valid_field?(%{"name" => name, "value" => value}) do - is_binary(name) && is_binary(value) + value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) + + is_binary(name) && + is_binary(value) && + String.length(name) <= 255 && + String.length(value) <= value_limit end defp valid_field?(_), do: false diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index a3eadde16..992a692f0 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,6 +325,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] + value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) + + long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => "foo", "value" => long_str}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + + fields = [%{"name" => long_str, "value" => "bar"}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + Pleroma.Config.put([:instance, :max_account_fields], 1) fields = [ @@ -332,12 +352,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] - conn = - conn - |> assign(:user, user) - |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) - - assert %{"error" => "Invalid request"} == json_response(conn, 403) + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) end end end From 2c35d4b0b04e58368c51f2828536d295f72839a2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 1 Aug 2019 15:09:15 +0700 Subject: [PATCH 26/66] Add configurable account field name length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 3 ++- .../mastodon_api_controller/update_credentials_test.exs | 9 ++++++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 21f4861f2..4fd241e90 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index fbb506093..6744f5879 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9e4d381f8..e54243f06 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,11 +308,12 @@ defmodule Pleroma.User.Info do end defp valid_field?(%{"name" => name, "value" => value}) do + name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255) value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) is_binary(name) && is_binary(value) && - String.length(name) <= 255 && + String.length(name) <= name_limit && String.length(value) <= value_limit end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 992a692f0..e75f25d51 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,11 +325,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] + name_limit = Pleroma.Config.get([:instance, :account_field_name_length]) value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) - long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + long_value = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() - fields = [%{"name" => "foo", "value" => long_str}] + fields = [%{"name" => "foo", "value" => long_value}] assert %{"error" => "Invalid request"} == conn @@ -337,7 +338,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) |> json_response(403) - fields = [%{"name" => long_str, "value" => "bar"}] + long_name = Enum.map(0..name_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => long_name, "value" => "bar"}] assert %{"error" => "Invalid request"} == conn From f7bbf99caade7f06756e95e3a4e2f0e4d3e76579 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 6 Aug 2019 18:21:25 +0700 Subject: [PATCH 27/66] Use info.fields instead of source_data for remote users --- lib/pleroma/html.ex | 28 ++++++++++++++++++++++ lib/pleroma/user/info.ex | 6 ++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 10 +++++--- lib/pleroma/web/activity_pub/views/user_view.ex | 7 +++++- .../web/mastodon_api/mastodon_api_controller.ex | 13 ++++------ lib/pleroma/web/mastodon_api/views/account_view.ex | 18 ++++++++++---- lib/pleroma/web/twitter_api/views/user_view.ex | 10 +++++++- test/web/activity_pub/transmogrifier_test.exs | 12 +++++----- test/web/activity_pub/views/user_view_test.exs | 17 +++++-------- .../update_credentials_test.exs | 9 ++++--- 10 files changed, 91 insertions(+), 39 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 2fae7281c..bf2000d90 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -280,3 +280,31 @@ defmodule Pleroma.HTML.Transform.MediaProxy do def scrub({_tag, children}), do: children def scrub(text), do: text end + +defmodule Pleroma.HTML.Scrubber.LinksOnly do + @moduledoc """ + An HTML scrubbing policy which limits to links only. + """ + + @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) + + require HtmlSanitizeEx.Scrubber.Meta + alias HtmlSanitizeEx.Scrubber.Meta + + Meta.remove_cdata_sections_before_scrub() + Meta.strip_comments() + + # links + Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes) + + Meta.allow_tag_with_this_attribute_values("a", "rel", [ + "tag", + "nofollow", + "noopener", + "noreferrer", + "me" + ]) + + Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.strip_everything_not_covered() +end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e54243f06..ada9fb689 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -50,6 +50,7 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: []) + field(:raw_fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -270,8 +271,10 @@ defmodule Pleroma.User.Info do :follower_count, :following_count, :hide_follows, + :fields, :hide_followers ]) + |> validate_fields() end def profile_update(info, params) do @@ -288,6 +291,7 @@ defmodule Pleroma.User.Info do :show_role, :skip_thread_containment, :fields, + :raw_fields, :pleroma_settings_store ]) |> validate_fields() @@ -415,7 +419,7 @@ defmodule Pleroma.User.Info do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] - def fields(%{source_data: %{"attachment" => attachment}}) do + def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 225c34875..2be2e3294 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,13 +598,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false - attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) + attachment = get_in(new_user_data, [:info, :source_data, "attachment"]) || [] + + fields = + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) - |> Map.put(:info, %{banner: banner, locked: locked}) - |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) + |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor |> User.upgrade_changeset(update_data) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index b2a22478d..7be734b26 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -83,8 +83,13 @@ defmodule Pleroma.Web.ActivityPub.UserView do fields = user.info |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) |> Enum.map(&Map.put(&1, "type", "PropertyValue")) - |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e79a02caa..e8fac8880 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -137,7 +137,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") user_info_emojis = - ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + user.info + |> Map.get(:emoji, []) + |> Enum.concat(Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = @@ -157,16 +159,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) |> add_if_present(params, "default_scope", :default_scope) |> add_if_present(params, "fields", :fields, fn fields -> - fields = - Enum.map(fields, fn field -> - %{ - "name" => Formatter.html_escape(field["name"], "text/plain"), - "value" => Formatter.html_escape(field["value"], "text/plain") - } - end) + fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) {:ok, fields} end) + |> add_if_present(params, "fields", :raw_fields) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index d2f3986ff..a2297a8e8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,11 +93,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - fields = User.Info.fields(user.info) - fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) + + raw_fields = Map.get(user.info, :raw_fields, []) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) - relationship = render("relationship.json", %{user: opts[:for], target: user}) %{ @@ -117,12 +125,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header: header, header_static: header, emojis: emojis, - fields: fields_html, + fields: fields, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, - fields: fields, + fields: raw_fields, pleroma: %{} }, diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 3681773be..8a7d2fc72 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,7 +74,15 @@ defmodule Pleroma.Web.TwitterAPI.UserView do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - fields = User.Info.fields(user.info) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) data = %{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 05ec09ec1..7e2c8769d 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -518,9 +518,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = User.get_cached_by_ap_id(activity.actor) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo1", "value" => "bar1"} ] update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() @@ -539,9 +539,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = User.get_cached_by_ap_id(user.ap_id) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} ] end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index a2aa52381..fb7fd9e79 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -24,21 +24,16 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "Renders profile fields" do fields = [ - %{"name" => "foo", "value" => "bar"}, - %{"name" => "website", "value" => "cofe.my"} + %{"name" => "foo", "value" => "bar"} ] - user = insert(:user, info: %{fields: fields}) + {:ok, user} = + insert(:user) + |> User.upgrade_changeset(%{info: %{fields: fields}}) + |> User.update_and_set_cache() assert %{ - "attachment" => [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{ - "name" => "website", - "type" => "PropertyValue", - "value" => "cofe.my" - } - ] + "attachment" => [%{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}] } = UserView.render("user.json", %{user: user}) end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index e75f25d51..dd443495b 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -305,7 +305,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do user = insert(:user) fields = [ - %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo", "value" => ""}, %{"name" => "link", "value" => "cofe.io"} ] @@ -316,12 +316,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do |> json_response(200) assert account["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "foo", "value" => "bar"}, %{"name" => "link", "value" => "cofe.io"} ] assert account["source"]["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{ + "name" => "foo", + "value" => "" + }, %{"name" => "link", "value" => "cofe.io"} ] From e457fcc47971df6c76c3da096e6b45c2972e4029 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:14:22 +0700 Subject: [PATCH 28/66] Add `:max_remote_account_fields` config option --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user.ex | 4 ++-- lib/pleroma/user/info.ex | 11 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 18 ++++++++++++++++++ 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4fd241e90..3937f7e70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + max_remote_account_fields: 10, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index 6744f5879..b4bdbd078 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b67743846..faa1e3d50 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -199,12 +199,12 @@ defmodule Pleroma.User do |> validate_length(:name, min: 1, max: name_limit) end - def upgrade_changeset(struct, params \\ %{}) do + def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now()) - info_cng = User.Info.user_upgrade(struct.info, params[:info]) + info_cng = User.Info.user_upgrade(struct.info, params[:info], remote?) struct |> cast(params, [ diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ada9fb689..47e7df911 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -256,11 +256,13 @@ defmodule Pleroma.User.Info do :hide_followers, :hide_follows, :follower_count, + :fields, :following_count ]) + |> validate_fields(true) end - def user_upgrade(info, params) do + def user_upgrade(info, params, remote? \\ false) do info |> cast(params, [ :ap_enabled, @@ -274,7 +276,7 @@ defmodule Pleroma.User.Info do :fields, :hide_followers ]) - |> validate_fields() + |> validate_fields(remote?) end def profile_update(info, params) do @@ -297,8 +299,9 @@ defmodule Pleroma.User.Info do |> validate_fields() end - def validate_fields(changeset) do - limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + def validate_fields(changeset, remote? \\ false) do + limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields + limit = Pleroma.Config.get([:instance, limit_name], 0) changeset |> validate_length(:fields, max: limit) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c9520..7bb7740bf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1016,6 +1016,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "url" => [%{"href" => data["image"]["url"]}] } + fields = + data + |> Map.get("attachment", []) + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) @@ -1025,6 +1031,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ap_enabled: true, source_data: data, banner: banner, + fields: fields, locked: locked }, avatar: avatar, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2be2e3294..36340a3a1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -611,7 +611,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor - |> User.upgrade_changeset(update_data) + |> User.upgrade_changeset(update_data, true) |> User.update_and_set_cache() ActivityPub.update(%{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7e2c8769d..d8fbcd628 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -543,6 +543,24 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do %{"name" => "foo", "value" => "updated"}, %{"name" => "foo1", "value" => "updated"} ] + + Pleroma.Config.put([:instance, :max_remote_account_fields], 2) + + update_data = + put_in(update_data, ["object", "attachment"], [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo11", "type" => "PropertyValue", "value" => "bar11"}, + %{"name" => "foo22", "type" => "PropertyValue", "value" => "bar22"} + ]) + + {:ok, _} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} + ] end test "it works for incoming update activities which lock the account" do From 672fcbc7b716f18346a17845d05c286b45dca5f3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:48:05 +0700 Subject: [PATCH 29/66] Limit custom fields for old remote users --- lib/pleroma/user/info.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 47e7df911..45a39924b 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -423,9 +423,12 @@ defmodule Pleroma.User.Info do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do + limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0) + attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + |> Enum.take(limit) end def fields(%{fields: fields}), do: fields From a2e1db56323d0f306ee42a1f58471eb55c8c1e68 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 8 Aug 2019 13:20:35 +0700 Subject: [PATCH 30/66] Increase max account fields limits --- config/config.exs | 4 ++-- docs/config.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 3937f7e70..7279b43b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,8 +255,8 @@ config :pleroma, :instance, dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, - max_account_fields: 4, - max_remote_account_fields: 10, + max_account_fields: 10, + max_remote_account_fields: 20, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index b4bdbd078..7e4e96db0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,8 +132,8 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. From 4b7f1c6995ca49c782e3e29d14245f18d4d11430 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 20:46:05 +0700 Subject: [PATCH 31/66] Improve digest email template --- lib/mix/tasks/pleroma/digest.ex | 12 +- lib/pleroma/emails/user_email.ex | 17 + lib/pleroma/web/templates/email/digest.html.eex | 581 +++++++++++++++++++++++- lib/pleroma/web/views/email_view.ex | 10 + test/mix/tasks/pleroma.digest_test.exs | 2 +- 5 files changed, 602 insertions(+), 20 deletions(-) diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index 81c207e10..430116a50 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -27,7 +27,15 @@ defmodule Mix.Tasks.Pleroma.Digest do patched_user = %{user | last_digest_emailed_at: last_digest_emailed_at} - _user = Pleroma.DigestEmailWorker.perform(patched_user) - Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do + {:ok, _} = Pleroma.Emails.Mailer.deliver(email) + + Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + else + _ -> + Mix.shell().info( + "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}" + ) + end end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 49046bb8b..bf6b811b1 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -123,6 +123,11 @@ defmodule Pleroma.Emails.UserEmail do end) with [_ | _] = mentions <- new_notifications.mentions do + mentions = + Enum.map(mentions, fn mention -> + update_in(mention.object.data["content"], &format_links/1) + end) + html_data = %{ instance: instance_name(), user: user, @@ -131,17 +136,29 @@ defmodule Pleroma.Emails.UserEmail do unsubscribe_link: unsubscribe_url(user, "digest") } + logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + new() |> to(recipient(user)) |> from(sender()) |> subject("Your digest from #{instance_name()}") + |> put_layout(false) |> render_body("digest.html", html_data) + |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) else _ -> nil end end + defp format_links(str) do + re = ~r//iU + + String.replace(str, re, fn link -> + String.replace(link, "Hey <%= @user.nickname %>, here is what you've missed! + -

New Mentions:

-
    -<%= for %{data: mention, object: object, from: from} <- @mentions do %> -
  • <%= link from.nickname, to: mention.activity.actor %>: <%= raw object.data["content"] %>
    • -<% end %> -
+ -<%= if @followers != [] do %> -

<%= length(@followers) %> New Followers:

-
    -<%= for %{data: follow, from: from} <- @followers do %> -
  • <%= link from.nickname, to: follow.activity.actor %>
    • -<% end %> -
-<% end %> + + + + + + + + <%= @email.subject %>< + + + + + + + + + + + + + + + + + + + diff --git a/lib/pleroma/web/views/email_view.ex b/lib/pleroma/web/views/email_view.ex index b63eb162c..b506a234b 100644 --- a/lib/pleroma/web/views/email_view.ex +++ b/lib/pleroma/web/views/email_view.ex @@ -2,4 +2,14 @@ defmodule Pleroma.Web.EmailView do use Pleroma.Web, :view import Phoenix.HTML import Phoenix.HTML.Link + + def avatar_url(user) do + Pleroma.User.avatar_url(user) + end + + def format_date(date) when is_binary(date) do + date + |> Timex.parse!("{ISO:Extended:Z}") + |> Timex.format!("{Mshort} {D}, {YYYY} {h24}:{m}") + end end diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs index 595f64ed7..4bfa1fb93 100644 --- a/test/mix/tasks/pleroma.digest_test.exs +++ b/test/mix/tasks/pleroma.digest_test.exs @@ -44,7 +44,7 @@ defmodule Mix.Tasks.Pleroma.DigestTest do assert_email_sent( to: {user2.name, user2.email}, - html_body: ~r/new mentions:/i + html_body: ~r/here is what you've missed!/i ) end end From df81abb68c46e36dfb2af7aab77e0e57a1a1eb28 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 14 Aug 2019 15:55:43 +0200 Subject: [PATCH 32/66] Conversations: Use correct oauth paths for extended api. --- lib/pleroma/web/router.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9759268f9..1eb6f7b9d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -263,9 +263,13 @@ defmodule Pleroma.Web.Router do pipe_through(:authenticated_api) scope [] do - pipe_through(:oauth_write) + pipe_through(:oauth_read) get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) get("/conversations/:id", PleromaAPIController, :conversation) + end + + scope [] do + pipe_through(:oauth_write) patch("/conversations/:id", PleromaAPIController, :update_conversation) end end From f73212b2a36deef631716f3c8a80d7da11cec759 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 14 Aug 2019 15:56:15 +0200 Subject: [PATCH 33/66] Conversation: Render new participation on update. --- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index b5c3d2728..6d74d418e 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do |> Participation.get() with true <- user.id == participation.user_id, - {:ok, _} <- Participation.set_recipients(participation, recipients) do + {:ok, participation} <- Participation.set_recipients(participation, recipients) do conn |> put_view(ConversationView) |> render("participation.json", %{participation: participation, for: user}) From c9970feee20f25375223e5f4a32bdbcff7b98607 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 21:03:25 +0700 Subject: [PATCH 34/66] Fix compatibility with Elixir 1.8 --- lib/pleroma/emails/user_email.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index bf6b811b1..3b5e64019 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -154,7 +154,7 @@ defmodule Pleroma.Emails.UserEmail do defp format_links(str) do re = ~r//iU - String.replace(str, re, fn link -> + Regex.replace(re, str, fn link -> String.replace(link, " Date: Wed, 14 Aug 2019 17:01:11 +0200 Subject: [PATCH 35/66] Conversations: Load relations in one query. --- lib/pleroma/conversation/participation.ex | 16 +++++++++++++--- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 4 +--- test/conversation/participation_test.exs | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index d17b6f7c5..ea5b9fe17 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -94,10 +94,20 @@ defmodule Pleroma.Conversation.Participation do |> Enum.filter(& &1.last_activity_id) end - def get(nil), do: nil + def get(_, _ \\ []) + def get(nil, _), do: nil - def get(id) do - Repo.get(__MODULE__, id) + def get(id, params) do + query = + if preload = params[:preload] do + from(p in __MODULE__, + preload: ^preload + ) + else + __MODULE__ + end + + Repo.get(query, id) end def set_recipients(participation, user_ids) do diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 6d74d418e..b6d2bf86b 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Conversation.Participation - alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView @@ -34,8 +33,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do participation = participation_id - |> Participation.get() - |> Repo.preload(:conversation) + |> Participation.get(preload: [:conversation]) if user.id == participation.user_id do activities = diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 7958e8e89..a27167d42 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -8,6 +8,20 @@ defmodule Pleroma.Conversation.ParticipationTest do alias Pleroma.Conversation.Participation alias Pleroma.Web.CommonAPI + test "getting a participation will also preload things" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + [participation] = Participation.for_user(user) + + participation = Participation.get(participation.id, preload: [:conversation]) + + assert %Pleroma.Conversation{} = participation.conversation + end + test "for a new conversation, it sets the recipents of the participation" do user = insert(:user) other_user = insert(:user) From 51bdf0cab6dc96bfd48a6d98d9f21584b42c0e44 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:55:17 -0400 Subject: [PATCH 36/66] use default child_specs --- lib/pleroma/application.ex | 185 +++++++++------------------- lib/pleroma/captcha/captcha.ex | 2 +- lib/pleroma/config/transfer_task.ex | 2 +- lib/pleroma/emoji.ex | 2 +- lib/pleroma/flake_id.ex | 2 +- lib/pleroma/gopher/server.ex | 2 +- lib/pleroma/scheduled_activity_worker.ex | 2 +- lib/pleroma/stats.ex | 4 +- lib/pleroma/web/chat_channel.ex | 4 +- lib/pleroma/web/federator/retry_queue.ex | 2 +- lib/pleroma/web/oauth/token/clean_worker.ex | 3 +- lib/pleroma/web/streamer.ex | 2 +- test/config/transfer_task_test.exs | 4 +- 13 files changed, 73 insertions(+), 143 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00b06f723..3bb0718e4 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,11 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do + import Cachex.Spec use Application @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @env Mix.env() + def name, do: @name def version, do: @version def named_version, do: @name <> " " <> @version @@ -21,116 +24,25 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do - import Cachex.Spec - Pleroma.Config.DeprecationWarnings.warn() setup_instrumenters() # Define workers and child supervisors to be supervised children = [ - # Start the Ecto repository - %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor}, - %{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}}, - %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}}, - %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}}, - %{ - id: :cachex_used_captcha_cache, - start: - {Cachex, :start_link, - [ - :used_captcha_cache, - [ - ttl_interval: - :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) - ] - ]} - }, - %{ - id: :cachex_user, - start: - {Cachex, :start_link, - [ - :user_cache, - [ - default_ttl: 25_000, - ttl_interval: 1000, - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_object, - start: - {Cachex, :start_link, - [ - :object_cache, - [ - default_ttl: 25_000, - ttl_interval: 1000, - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_rich_media, - start: - {Cachex, :start_link, - [ - :rich_media_cache, - [ - default_ttl: :timer.minutes(120), - limit: 5000 - ] - ]} - }, - %{ - id: :cachex_scrubber, - start: - {Cachex, :start_link, - [ - :scrubber_cache, - [ - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_idem, - start: - {Cachex, :start_link, - [ - :idempotency_cache, - [ - expiration: - expiration( - default: :timer.seconds(6 * 60 * 60), - interval: :timer.seconds(60) - ), - limit: 2500 - ] - ]} - }, - %{id: Pleroma.FlakeId, start: {Pleroma.FlakeId, :start_link, []}}, - %{ - id: Pleroma.ScheduledActivityWorker, - start: {Pleroma.ScheduledActivityWorker, :start_link, []} - } + Pleroma.Repo, + Pleroma.Config.TransferTask, + Pleroma.Emoji, + Pleroma.Captcha, + Pleroma.FlakeId, + Pleroma.ScheduledActivityWorker ] ++ + cachex_children() ++ hackney_pool_children() ++ [ - %{ - id: Pleroma.Web.Federator.RetryQueue, - start: {Pleroma.Web.Federator.RetryQueue, :start_link, []} - }, - %{ - id: Pleroma.Web.OAuth.Token.CleanWorker, - start: {Pleroma.Web.OAuth.Token.CleanWorker, :start_link, []} - }, - %{ - id: Pleroma.Stats, - start: {Pleroma.Stats, :start_link, []} - }, + Pleroma.Web.Federator.RetryQueue, + Pleroma.Web.OAuth.Token.CleanWorker, + Pleroma.Stats, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, @@ -147,16 +59,11 @@ defmodule Pleroma.Application do restart: :temporary } ] ++ - streamer_child() ++ - chat_child() ++ + streamer_child(@env) ++ + chat_child(@env, chat_enabled?()) ++ [ - # Start the endpoint when the application starts - %{ - id: Pleroma.Web.Endpoint, - start: {Pleroma.Web.Endpoint, :start_link, []}, - type: :supervisor - }, - %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}} + Pleroma.Web.Endpoint, + Pleroma.Gopher.Server ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -201,28 +108,46 @@ defmodule Pleroma.Application do end end - if Pleroma.Config.get(:env) == :test do - defp streamer_child, do: [] - defp chat_child, do: [] - else - defp streamer_child do - [%{id: Pleroma.Web.Streamer, start: {Pleroma.Web.Streamer, :start_link, []}}] - end - - defp chat_child do - if Pleroma.Config.get([:chat, :enabled]) do - [ - %{ - id: Pleroma.Web.ChatChannel.ChatChannelState, - start: {Pleroma.Web.ChatChannel.ChatChannelState, :start_link, []} - } - ] - else - [] - end - end + defp cachex_children do + [ + build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), + build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), + build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), + build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), + build_cachex("scrubber", limit: 2500), + build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500) + ] end + defp idempotency_expiration, + do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + + defp seconds_valid_interval, + do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + + defp build_cachex(type, opts), + do: %{ + id: String.to_atom("cachex_" <> type), + start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, + type: :worker + } + + defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + + defp streamer_child(:test), do: [] + + defp streamer_child(_) do + [Pleroma.Web.Streamer] + end + + defp chat_child(:test, _), do: [] + + defp chat_child(_env, true) do + [Pleroma.Web.ChatChannel.ChatChannelState] + end + + defp chat_child(_, _), do: [] + defp hackney_pool_children do for pool <- enabled_hackney_pools() do options = Pleroma.Config.get([:hackney_pools, pool]) diff --git a/lib/pleroma/captcha/captcha.ex b/lib/pleroma/captcha/captcha.ex index a73b87251..c2765a5b8 100644 --- a/lib/pleroma/captcha/captcha.ex +++ b/lib/pleroma/captcha/captcha.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Captcha do use GenServer @doc false - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 7799b2a78..3214c9951 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Config.TransferTask do use Task alias Pleroma.Web.AdminAPI.Config - def start_link do + def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo) :ignore diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 052501642..66e20f0e4 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Emoji do @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @doc false - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index ca0610abc..47d61ca5f 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -98,7 +98,7 @@ defmodule Pleroma.FlakeId do def autogenerate, do: get() # -- GenServer API - def start_link do + def start_link(_) do :gen_server.start_link({:local, :flake}, __MODULE__, [], []) end diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index b3319e137..d4e4f3e55 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Gopher.Server do use GenServer require Logger - def start_link do + def start_link(_) do config = Pleroma.Config.get(:gopher, []) ip = Keyword.get(config, :ip, {0, 0, 0, 0}) port = Keyword.get(config, :port, 1234) diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 65b38622f..8578cab5e 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -16,7 +16,7 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, nil) end diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 5b242927b..101effbe4 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Stats do alias Pleroma.Repo alias Pleroma.User - def start_link do + use Agent + + def start_link(_) do agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__) spawn(fn -> schedule_update() end) agent diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex index f63f4bda1..b543909f1 100644 --- a/lib/pleroma/web/chat_channel.ex +++ b/lib/pleroma/web/chat_channel.ex @@ -33,9 +33,11 @@ defmodule Pleroma.Web.ChatChannel do end defmodule Pleroma.Web.ChatChannel.ChatChannelState do + use Agent + @max_messages 20 - def start_link do + def start_link(_) do Agent.start_link(fn -> %{max_id: 1, messages: []} end, name: __MODULE__) end diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex index 3db948c2e..9eab8c218 100644 --- a/lib/pleroma/web/federator/retry_queue.ex +++ b/lib/pleroma/web/federator/retry_queue.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}} end - def start_link do + def start_link(_) do enabled = if Pleroma.Config.get(:env) == :test, do: true, diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index dca852449..e39a4986a 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @moduledoc """ The module represents functions to clean an expired oauth tokens. """ + use GenServer # 10 seconds @start_interval 10_000 @@ -18,7 +19,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do alias Pleroma.Web.OAuth.Token - def start_link, do: GenServer.start_link(__MODULE__, nil) + def start_link(_), do: GenServer.start_link(__MODULE__, nil) def init(_) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 9ee331030..e66378ceb 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.Streamer do @keepalive_interval :timer.seconds(30) - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, %{}, name: __MODULE__) end diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index dbeadbe87..4455a4d47 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -31,7 +31,7 @@ defmodule Pleroma.Config.TransferTaskTest do value: [live: 15, com: 35] }) - Pleroma.Config.TransferTask.start_link() + Pleroma.Config.TransferTask.start_link([]) assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3] assert Application.get_env(:idna, :test_key) == [live: 15, com: 35] @@ -50,7 +50,7 @@ defmodule Pleroma.Config.TransferTaskTest do }) assert ExUnit.CaptureLog.capture_log(fn -> - Pleroma.Config.TransferTask.start_link() + Pleroma.Config.TransferTask.start_link([]) end) =~ "updating env causes error, key: \"undefined_atom_key\", error: %ArgumentError{message: \"argument error\"}" end From 15ef521009f4c232f417ca9164c6be3f4ee4e018 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:57:50 -0400 Subject: [PATCH 37/66] Isolate OAuth.Token.CleanWorker --- lib/pleroma/application.ex | 9 +++++++++ lib/pleroma/web/oauth/token/clean_worker.ex | 25 +++++++++---------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3bb0718e4..c460a3bc5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Application do restart: :temporary } ] ++ + oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ [ @@ -134,12 +135,20 @@ defmodule Pleroma.Application do defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp oauth_cleanup_enabled?, + do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) + defp streamer_child(:test), do: [] defp streamer_child(_) do [Pleroma.Web.Streamer] end + defp oauth_cleanup_child(true), + do: [Pleroma.Web.OAuth.Token.CleanWorker] + + defp oauth_cleanup_child(_), do: [] + defp chat_child(:test, _), do: [] defp chat_child(_env, true) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index e39a4986a..f50098302 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -8,35 +8,28 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do """ use GenServer - # 10 seconds - @start_interval 10_000 + @ten_seconds 10_000 + @one_day 86_400_000 + @interval Pleroma.Config.get( - # 24 hours [:oauth2, :clean_expired_tokens_interval], - 86_400_000 + @one_day ) - @queue :background alias Pleroma.Web.OAuth.Token - def start_link(_), do: GenServer.start_link(__MODULE__, nil) + def start_link(_), do: GenServer.start_link(__MODULE__, %{}) def init(_) do - if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do - Process.send_after(self(), :perform, @start_interval) - {:ok, nil} - else - :ignore - end + Process.send_after(self(), :perform, @ten_seconds) + {:ok, nil} end @doc false def handle_info(:perform, state) do + Token.delete_expired_tokens() + Process.send_after(self(), :perform, @interval) - PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) {:noreply, state} end - - # Job Worker Callbacks - def perform(:clean), do: Token.delete_expired_tokens() end From 574856ef01fae7ea411ec363929ab9a22d76a65d Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:58:32 -0400 Subject: [PATCH 38/66] streamline Streamer pings --- lib/pleroma/web/streamer.ex | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index e66378ceb..bbaddd852 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -35,28 +35,21 @@ defmodule Pleroma.Web.Streamer do end def init(args) do - spawn(fn -> - # 30 seconds - Process.sleep(@keepalive_interval) - GenServer.cast(__MODULE__, %{action: :ping}) - end) + Process.send_after(self(), %{action: :ping}, @keepalive_interval) {:ok, args} end - def handle_cast(%{action: :ping}, topics) do - Map.values(topics) + def handle_info(%{action: :ping}, topics) do + topics + |> Map.values() |> List.flatten() |> Enum.each(fn socket -> Logger.debug("Sending keepalive ping") send(socket.transport_pid, {:text, ""}) end) - spawn(fn -> - # 30 seconds - Process.sleep(@keepalive_interval) - GenServer.cast(__MODULE__, %{action: :ping}) - end) + Process.send_after(self(), %{action: :ping}, @keepalive_interval) {:noreply, topics} end From d81f63845a71e5cc60d95007ecaa2aea52a90422 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:59:33 -0400 Subject: [PATCH 39/66] Implement Pleroma.Stats as GenServer --- lib/pleroma/stats.ex | 60 +++++++++++++++------- .../mastodon_api/mastodon_api_controller_test.exs | 4 +- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 101effbe4..a3b8a4d66 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -7,33 +7,56 @@ defmodule Pleroma.Stats do alias Pleroma.Repo alias Pleroma.User - use Agent + use GenServer + + @interval 1000 * 60 * 60 def start_link(_) do - agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__) - spawn(fn -> schedule_update() end) - agent + GenServer.start_link(__MODULE__, initial_data(), name: __MODULE__) + end + + def force_update do + GenServer.call(__MODULE__, :force_update) end def get_stats do - Agent.get(__MODULE__, fn {_, stats} -> stats end) + %{stats: stats} = GenServer.call(__MODULE__, :get_state) + + stats end def get_peers do - Agent.get(__MODULE__, fn {peers, _} -> peers end) + %{peers: peers} = GenServer.call(__MODULE__, :get_state) + + peers end - def schedule_update do - spawn(fn -> - # 1 hour - Process.sleep(1000 * 60 * 60) - schedule_update() - end) - - update_stats() + def init(args) do + Process.send_after(self(), :run_update, @interval) + {:ok, args} end - def update_stats do + def handle_call(:force_update, _from, _state) do + new_stats = get_stat_data() + {:reply, new_stats, new_stats} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_info(:run_update, _state) do + new_stats = get_stat_data() + + Process.send_after(self(), :run_update, @interval) + {:noreply, new_stats} + end + + defp initial_data do + %{peers: [], stats: %{}} + end + + defp get_stat_data do peers = from( u in User, @@ -54,8 +77,9 @@ defmodule Pleroma.Stats do user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id) - Agent.update(__MODULE__, fn _ -> - {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}} - end) + %{ + peers: peers, + stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + } end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 2febe8b3a..112e272f9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2624,7 +2624,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> Changeset.put_embed(:info, info_change) |> User.update_and_set_cache() - Pleroma.Stats.update_stats() + Pleroma.Stats.force_update() conn = get(conn, "/api/v1/instance") @@ -2642,7 +2642,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do insert(:user, %{local: false, nickname: "u@peer1.com"}) insert(:user, %{local: false, nickname: "u@peer2.com"}) - Pleroma.Stats.update_stats() + Pleroma.Stats.force_update() conn = get(conn, "/api/v1/instance/peers") From c43152f6c17a287a9fe4f2556ca20a140ea30248 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 14:01:11 -0400 Subject: [PATCH 40/66] fix formatting --- lib/pleroma/application.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c460a3bc5..aa673188f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -121,17 +121,17 @@ defmodule Pleroma.Application do end defp idempotency_expiration, - do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), - do: %{ - id: String.to_atom("cachex_" <> type), - start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, - type: :worker - } + do: %{ + id: String.to_atom("cachex_" <> type), + start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, + type: :worker + } defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) @@ -145,7 +145,7 @@ defmodule Pleroma.Application do end defp oauth_cleanup_child(true), - do: [Pleroma.Web.OAuth.Token.CleanWorker] + do: [Pleroma.Web.OAuth.Token.CleanWorker] defp oauth_cleanup_child(_), do: [] From 626e094589689681845d8057f8abe6ab3cd52f69 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 18:53:18 +0000 Subject: [PATCH 41/66] MRF: fix up unserializable option lists in describe implementations --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 3 ++- lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 3 ++- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 3 ++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 358287096..835dbc14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user - MRF: ensure that subdomain_match calls are case-insensitive +- MRF: fix use of unserializable keyword lists in describe() implementations ### Added - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 9863454fa..b3c742954 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -92,5 +92,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do def filter(message), do: {:ok, message} @impl true - def describe, do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} + def describe, + do: {:ok, %{mrf_hellthread: Pleroma.Config.get(:mrf_hellthread) |> Enum.into(%{})}} end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 0ae9397ed..5a809a321 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -46,5 +46,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do def filter(object), do: {:ok, object} @impl true - def describe, do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} + def describe, + do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 74da8d57e..4eaea00d8 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -32,5 +32,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do def filter(message), do: {:ok, message} - def describe, do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary)}} + def describe, + do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Enum.into(%{})}} end From 5bb418a90d9efb1fa889028080c5de3a929ff2cc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 19:00:48 +0000 Subject: [PATCH 42/66] activitypub: publisher: add (request-target) to http signature when POSTing --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/publisher.ex | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 358287096..5a6150d98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Rich Media: The crawled URL is now spliced into the rich media data. - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification. - ActivityPub S2S: remote user deletions now work the same as local user deletions. +- ActivityPub S2S: POST requests are now signed with `(request-target)` pseudo-header. - Not being able to access the Mastodon FE login page on private instances - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 46edab0bd..987a25377 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -46,7 +46,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do Logger.info("Federating #{id} to #{inbox}") - host = URI.parse(inbox).host + uri = URI.parse(inbox) + host = uri.host + path = uri.path digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) @@ -56,6 +58,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do signature = Pleroma.Signature.sign(actor, %{ + "(request-target)": "post #{path}", host: host, "content-length": byte_size(json), digest: digest, From 1754f8ce6d9ee896a57961a354001b713c620cd5 Mon Sep 17 00:00:00 2001 From: kaniini Date: Wed, 14 Aug 2019 19:05:44 +0000 Subject: [PATCH 43/66] Apply suggestion to lib/pleroma/web/activity_pub/publisher.ex --- lib/pleroma/web/activity_pub/publisher.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 987a25377..262529b84 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -46,9 +46,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do Logger.info("Federating #{id} to #{inbox}") - uri = URI.parse(inbox) - host = uri.host - path = uri.path + %{host: host, path: path} = URI.parse(inbox) digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) From a6a814420ded3973b271d04b29b4d6ad24b6bdf7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 14 Aug 2019 22:48:44 +0200 Subject: [PATCH 44/66] html.ex: Allow sub and sup elements by default Closes: https://git.pleroma.social/pleroma/pleroma/issues/1191 --- lib/pleroma/html.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 2fae7281c..06e60cba3 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -203,6 +203,8 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("p", []) Meta.allow_tag_with_these_attributes("pre", []) Meta.allow_tag_with_these_attributes("strong", []) + Meta.allow_tag_with_these_attributes("sub", []) + Meta.allow_tag_with_these_attributes("sup", []) Meta.allow_tag_with_these_attributes("u", []) Meta.allow_tag_with_these_attributes("ul", []) From a9e75fa6a4ede24fbd4549d4deb06edf368e7c52 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 00:43:02 +0300 Subject: [PATCH 45/66] Add a task to benchmark timeline rendering --- lib/mix/tasks/pleroma/benchmark.ex | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 5222cce80..a45940bf3 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -26,4 +26,28 @@ defmodule Mix.Tasks.Pleroma.Benchmark do end }) end + + def run(["render_timeline", nickname]) do + start_pleroma() + user = Pleroma.User.get_by_nickname(nickname) + + activities = + %{} + |> Map.put("type", ["Create", "Announce"]) + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() + |> Enum.reverse() + + Benchee.run(%{ + "render_timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }) + end end From bd5ad0af787e65bc05b7df64ef41c414900085af Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 00:47:30 +0300 Subject: [PATCH 46/66] Cache follow state --- lib/pleroma/user.ex | 22 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- lib/pleroma/web/activity_pub/utils.ex | 9 +++++++-- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 +++--- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b67743846..a1040fe71 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -132,6 +132,28 @@ defmodule Pleroma.User do |> Map.put(:follower_count, follower_count) end + def follow_state(%User{} = user, %User{} = target) do + follow_activity = Utils.fetch_latest_follow(user, target) + + if follow_activity, + do: follow_activity.data["state"], + # Ideally this would be nil, but then Cachex does not commit the value + else: false + end + + def get_cached_follow_state(user, target) do + key = "follow_state:#{user.ap_id}|#{target.ap_id}" + Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end) + end + + def set_follow_state_cache(user_ap_id, target_ap_id, state) do + Cachex.put( + :user_cache, + "follow_state:#{user_ap_id}|#{target_ap_id}", + state + ) + end + def set_info_cache(user, args) do Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user, args)) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c9520..01052846f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -388,7 +388,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def follow(follower, followed, activity_id \\ nil, local \\ true) do with data <- make_follow_data(follower, followed, activity_id), {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.set_follow_state_cache(follower.ap_id, followed.ap_id, activity.data["state"]) do {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index fc5305c58..1c3058658 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -374,6 +374,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do [state, actor, object] ) + User.set_follow_state_cache(actor, object, state) activity = Activity.get_by_id(activity.id) {:ok, activity} rescue @@ -382,12 +383,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do end end - def update_follow_state(%Activity{} = activity, state) do + def update_follow_state( + %Activity{data: %{"actor" => actor, "object" => object}} = activity, + state + ) do with new_data <- activity.data |> Map.put("state", state), changeset <- Changeset.change(activity, data: new_data), - {:ok, activity} <- Repo.update(changeset) do + {:ok, activity} <- Repo.update(changeset), + _ <- User.set_follow_state_cache(actor, object, state) do {:ok, activity} end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 72c092f25..0ef568f0f 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -37,11 +37,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do - follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target) + follow_state = User.get_cached_follow_state(user, target) requested = - if follow_activity && !User.following?(target, user) do - follow_activity.data["state"] == "pending" + if follow_state && !User.following?(user, target) do + follow_state == "pending" else false end From e8a8d50138f70240bf853ba67008dc19e75127e8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 01:01:13 +0300 Subject: [PATCH 47/66] Collect stats immediately after init --- lib/pleroma/stats.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index a3b8a4d66..df80fbaa4 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Stats do end def init(args) do - Process.send_after(self(), :run_update, @interval) + Process.send(self(), :run_update, []) {:ok, args} end From 2b94ae3b3911d78ee603163f7c8aa256ed65643f Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 15 Aug 2019 01:35:29 +0300 Subject: [PATCH 48/66] Do not check if actor is active when deleting a user --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++++----------- lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++++++------- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/admin_api/admin_api_controller.ex | 4 +-- lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 3 +- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccc36965..a7b776beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. +– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index f33d01429..a3f8bc945 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user, nil) + User.perform(:delete, user) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 14057a0e4..7d18f099e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,26 +1029,13 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - @spec delete(User.t()) :: :ok - def delete(%User{} = user, actor \\ nil), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) + def delete(%User{} = user), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user, actor) do - {:ok, _user} = ActivityPub.delete(user, actor: actor) + def perform(:delete, %User{} = user) do + {:ok, _user} = ActivityPub.delete(user) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1070,6 +1057,19 @@ defmodule Pleroma.User do Repo.delete(user) end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f669acb9..da873b7b0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -61,7 +61,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {recipients, to, cc} end - defp check_actor_is_active(actor) do + defp check_actor_is_active(true, _), do: :ok + + defp check_actor_is_active(false, actor) do if not is_nil(actor) do with user <- User.get_cached_by_ap_id(actor), false <- user.info.deactivated do @@ -119,10 +121,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop - def insert(map, local \\ true, fake \\ false) when is_map(map) do + def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), - :ok <- check_actor_is_active(map["actor"]), + :ok <- check_actor_is_active(bypass_actor_check, map["actor"]), {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), @@ -403,22 +405,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(data, opts \\ %{actor: nil, local: true}) - - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => opts[:actor] || ap_id, + "actor" => ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, - {:ok, activity} <- insert(data, true, true), + {:ok, activity} <- insert(data, true, true, true), :ok <- maybe_federate(activity) do {:ok, user} end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -430,7 +430,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, opts[:local], false), + {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b34ef73c0..5403b71d8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, local: false) do + {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 63c9a7d7f..2d3d0adc4 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + def user_delete(conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete(admin.ap_id) + |> User.delete() conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index ac2dc115c..b2f9f3946 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, local: false) do + {:ok, delete} <- ActivityPub.delete(object, false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index e2da8d84b..755c6005d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -999,10 +999,9 @@ defmodule Pleroma.UserTest do end test "it deletes deactivated user" do - admin = insert(:user, %{info: %{is_admin: true}}) {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() - assert {:ok, _} = User.delete(user, admin.ap_id) + assert {:ok, _} = User.delete(user) refute User.get_by_id(user.id) end From b27fafe161241c954b713281bebd6ffe1e990884 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 15 Aug 2019 01:42:43 +0300 Subject: [PATCH 49/66] Fix CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b776beb..161f88176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. -– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config @@ -39,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user +- ActivityPub: Deactivated user deletion ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) From a4a3e3becd5e008dbfa9a23157ae4b16a0652bce Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 17:37:30 +0300 Subject: [PATCH 50/66] Hide muted theads from home/public timelines unless `with_muted` is set --- lib/pleroma/activity.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 20 +++++++++++++------- lib/pleroma/web/streamer.ex | 6 +++--- test/web/activity_pub/activity_pub_test.exs | 23 +++++++++++++++++++++++ test/web/streamer_test.exs | 20 ++++++++++++++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index baf1e7722..35612c882 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -96,6 +96,7 @@ defmodule Pleroma.Activity do from([a] in query, left_join: tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), + as: :thread_mute, select: %Activity{a | thread_muted?: not is_nil(tm.id)} ) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c9520..defccade8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -790,14 +790,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted(query, %{"with_muted" => val}) when val in [true, "true", "1"], do: query - defp restrict_muted(query, %{"muting_user" => %User{info: info}}) do + defp restrict_muted(query, %{"muting_user" => %User{info: info}} = opts) do mutes = info.mutes - from( - activity in query, - where: fragment("not (? = ANY(?))", activity.actor, ^mutes), - where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes) - ) + query = + from([activity] in query, + where: fragment("not (? = ANY(?))", activity.actor, ^mutes), + where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes) + ) + + unless opts["skip_preload"] do + from([thread_mute: tm] in query, where: is_nil(tm)) + else + query + end end defp restrict_muted(query, _), do: query @@ -898,7 +904,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp maybe_set_thread_muted_field(query, opts) do query - |> Activity.with_set_thread_muted_field(opts["user"]) + |> Activity.with_set_thread_muted_field(opts["muting_user"] || opts["user"]) end defp maybe_order(query, %{order: :desc}) do diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index bbaddd852..826be2f9a 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -113,8 +113,7 @@ defmodule Pleroma.Web.Streamer do |> Map.get("#{topic}:#{item.user_id}", []) |> Enum.each(fn socket -> with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item), - false <- CommonAPI.thread_muted?(user, item.activity) do + true <- should_send?(user, item) do send( socket.transport_pid, {:text, represent_notification(socket.assigns[:user], item)} @@ -236,7 +235,8 @@ defmodule Pleroma.Web.Streamer do %{host: parent_host} <- URI.parse(parent.data["actor"]), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user) do + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do true else _ -> false diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d723f331f..0377d29f6 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -538,6 +538,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert Enum.member?(activities, activity_one) end + test "doesn't return thread muted activities" do + user = insert(:user) + activity_one = insert(:note_activity) + note_two = insert(:note, data: %{"context" => "suya.."}) + activity_two = insert(:note_activity, note: note_two) + + {:ok, _activity_two} = CommonAPI.add_mute(user, activity_two) + + assert [activity_one] = ActivityPub.fetch_activities([], %{"muting_user" => user}) + end + + test "returns thread muted activities when with_muted is set" do + user = insert(:user) + activity_one = insert(:note_activity) + note_two = insert(:note, data: %{"context" => "suya.."}) + activity_two = insert(:note_activity, note: note_two) + + {:ok, activity_two} = CommonAPI.add_mute(user, activity_two) + + assert [activity_two, activity_one] = + ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true}) + end + test "does include announces on request" do activity_three = insert(:note_activity) user = insert(:user) diff --git a/test/web/streamer_test.exs b/test/web/streamer_test.exs index d47b37efb..5b7fe44d4 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -414,6 +414,26 @@ defmodule Pleroma.Web.StreamerTest do Task.await(task) end + test "it doesn't send posts from muted threads" do + user = insert(:user) + user2 = insert(:user) + {:ok, user2, user, _activity} = CommonAPI.follow(user2, user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "super hot take"}) + + {:ok, activity} = CommonAPI.add_mute(user2, activity) + + task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + + Streamer.add_socket( + "user", + %{transport_pid: task.pid, assigns: %{user: user2}} + ) + + Streamer.stream("user", activity) + Task.await(task) + end + describe "direct streams" do setup do GenServer.start(Streamer, %{}, name: Streamer) From 1ad71592adb47762287aec8c36d0fca565c38362 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 17:41:26 +0300 Subject: [PATCH 51/66] Parallelize template rendering --- lib/mix/tasks/pleroma/benchmark.ex | 38 +++++++++++++++++------ lib/pleroma/web/mastodon_api/views/status_view.ex | 4 ++- lib/pleroma/web/web.ex | 18 +++++++++-- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a45940bf3..4cc634727 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -37,17 +37,37 @@ defmodule Mix.Tasks.Pleroma.Benchmark do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) + |> Map.put("limit", 80) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - Benchee.run(%{ - "render_timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end - }) + inputs = %{ + "One activity" => Enum.take_random(activities, 1), + "Ten activities" => Enum.take_random(activities, 10), + "Twenty activities" => Enum.take_random(activities, 20), + "Forty activities" => Enum.take_random(activities, 40), + "Eighty activities" => Enum.take_random(activities, 80) + } + + Benchee.run( + %{ + "Parallel rendering" => fn activities -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end, + "Standart rendering" => fn activities -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity, + parallel: false + }) + end + }, + inputs: inputs + ) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 492af1702..7e4e99280 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -70,12 +70,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) + parallel = unless is_nil(opts[:parallel]), do: opts[:parallel], else: true opts.activities |> safe_render_many( StatusView, "status.json", - Map.put(opts, :replied_to_activities, replied_to_activities) + Map.put(opts, :replied_to_activities, replied_to_activities), + parallel ) end diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index 687346554..bfb6c7287 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -66,9 +66,23 @@ defmodule Pleroma.Web do end @doc """ - Same as `render_many/4` but wrapped in rescue block. + Same as `render_many/4` but wrapped in rescue block and parallelized (unless disabled by passing false as a fifth argument). """ - def safe_render_many(collection, view, template, assigns \\ %{}) do + def safe_render_many(collection, view, template, assigns \\ %{}, parallel \\ true) + + def safe_render_many(collection, view, template, assigns, true) do + Enum.map(collection, fn resource -> + Task.async(fn -> + as = Map.get(assigns, :as) || view.__resource__ + assigns = Map.put(assigns, as, resource) + safe_render(view, template, assigns) + end) + end) + |> Enum.map(&Task.await(&1, :infinity)) + |> Enum.filter(& &1) + end + + def safe_render_many(collection, view, template, assigns, false) do Enum.map(collection, fn resource -> as = Map.get(assigns, :as) || view.__resource__ assigns = Map.put(assigns, as, resource) From fba3c16d205408bf6ba00e40bad7d8b5a7abe510 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 15 Aug 2019 20:36:20 +0300 Subject: [PATCH 52/66] Fix OAuth cleanup worker unconditionally starting !1576 removed enabled/disabled check from the worker, in favor of just not starting it in application.ex if disabled. However a line unconditionally starting the worker was removed --- lib/pleroma/application.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index aa673188f..25e56b9e2 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -41,7 +41,6 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Web.Federator.RetryQueue, - Pleroma.Web.OAuth.Token.CleanWorker, Pleroma.Stats, %{ id: :web_push_init, From 6a3b1a526eb1a3ea49aca7914ed7ba8736c52059 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 15 Aug 2019 15:34:41 -0500 Subject: [PATCH 53/66] max_body_size -> max_body_length, as it should be --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 4 ++-- test/reverse_proxy_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 1f98f215c..7518e4c3c 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -109,7 +109,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do + :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length, @max_body_length)) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -200,7 +200,7 @@ defmodule Pleroma.ReverseProxy do {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), - :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), + :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length)), {:ok, conn} <- chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index f4b7d6add..3a83c4c48 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -108,11 +108,11 @@ defmodule Pleroma.ReverseProxyTest do end end - test "max_body_size returns error if streaming body more than that option", %{conn: conn} do + test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_size: 30) + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) end) =~ "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" end From 158231cd20ceabf805ce49ae2b4d465e09e34d69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 16 Aug 2019 18:32:25 +0700 Subject: [PATCH 54/66] Add configurable colors and logo for the digest template --- config/config.exs | 11 ++ docs/config.md | 5 + lib/pleroma/emails/user_email.ex | 88 ++++++------- lib/pleroma/web/templates/email/digest.html.eex | 163 ++++++++++++------------ 4 files changed, 143 insertions(+), 124 deletions(-) diff --git a/config/config.exs b/config/config.exs index d2325edbc..4941953e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,6 +507,17 @@ config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false +config :pleroma, Pleroma.Emails.UserEmail, + logo: nil, + styling: %{ + link_color: "#d8a070", + background_color: "#2C3645", + content_background_color: "#1B2635", + header_color: "#d8a070", + text_color: "#b9b9ba", + text_muted_color: "#b9b9ba" + } + config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" config :pleroma, Pleroma.ScheduledActivity, diff --git a/docs/config.md b/docs/config.md index 703ef67dd..8360e640e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -548,6 +548,11 @@ Email notifications settings. - interval: Minimum interval between digest emails to one user - inactivity_threshold: Minimum user inactivity threshold +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + ## OAuth consumer mode OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 3b5e64019..40b67ff56 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -7,21 +7,21 @@ defmodule Pleroma.Emails.UserEmail do use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email} + alias Pleroma.Config + alias Pleroma.User alias Pleroma.Web.Endpoint alias Pleroma.Web.Router - defp instance_config, do: Pleroma.Config.get(:instance) - - defp instance_name, do: instance_config()[:name] + defp instance_name, do: Config.get([:instance, :name]) defp sender do - email = Keyword.get(instance_config(), :notify_email, instance_config()[:email]) + email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) {instance_name(), email} end defp recipient(email, nil), do: email defp recipient(email, name), do: {name, email} - defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name) + defp recipient(%User{} = user), do: recipient(user.email, user.name) def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) @@ -93,50 +93,54 @@ defmodule Pleroma.Emails.UserEmail do Includes Mentions and New Followers data If there are no mentions (even when new followers exist), the function will return nil """ - @spec digest_email(Pleroma.User.t()) :: Swoosh.Email.t() | nil + @spec digest_email(User.t()) :: Swoosh.Email.t() | nil def digest_email(user) do - new_notifications = - Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - |> Enum.reduce(%{followers: [], mentions: []}, fn - %{activity: %{data: %{"type" => "Create"}, actor: actor} = activity} = notification, - acc -> - new_mention = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } + notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - %{acc | mentions: [new_mention | acc.mentions]} + mentions = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Create")) + |> Enum.map(fn notification -> + object = Pleroma.Object.normalize(notification.activity) + object = update_in(object.data["content"], &format_links/1) - %{activity: %{data: %{"type" => "Follow"}, actor: actor} = activity} = notification, - acc -> - new_follower = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } - - %{acc | followers: [new_follower | acc.followers]} - - _, acc -> - acc + %{ + data: notification, + object: object, + from: User.get_by_ap_id(notification.activity.actor) + } end) - with [_ | _] = mentions <- new_notifications.mentions do - mentions = - Enum.map(mentions, fn mention -> - update_in(mention.object.data["content"], &format_links/1) - end) + followers = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Follow")) + |> Enum.map(fn notification -> + %{ + data: notification, + object: Pleroma.Object.normalize(notification.activity), + from: User.get_by_ap_id(notification.activity.actor) + } + end) + + unless Enum.empty?(mentions) do + styling = Config.get([__MODULE__, :styling]) + logo = Config.get([__MODULE__, :logo]) html_data = %{ instance: instance_name(), user: user, mentions: mentions, - followers: new_notifications.followers, - unsubscribe_link: unsubscribe_url(user, "digest") + followers: followers, + unsubscribe_link: unsubscribe_url(user, "digest"), + styling: styling } - logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + logo_path = + if is_nil(logo) do + Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + else + Path.join(Config.get([:instance, :static_dir]), logo) + end new() |> to(recipient(user)) @@ -145,17 +149,15 @@ defmodule Pleroma.Emails.UserEmail do |> put_layout(false) |> render_body("digest.html", html_data) |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) - else - _ -> - nil end end defp format_links(str) do re = ~r//iU + %{link_color: color} = Config.get([__MODULE__, :styling]) Regex.replace(re, str, fn link -> - String.replace(link, " user.id, "act" => %{"unsubscribe" => notifications_type}, "exp" => false} |> Pleroma.JWT.generate_and_sign!() |> Base.encode64() - Router.Helpers.subscription_url(Pleroma.Web.Endpoint, :unsubscribe, token) + Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end end diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex index 61d57093b..860df5f9c 100644 --- a/lib/pleroma/web/templates/email/digest.html.eex +++ b/lib/pleroma/web/templates/email/digest.html.eex @@ -21,7 +21,8 @@ } a { - color: #d8a070; + + color: <%= @styling.link_color %>; text-decoration: none; } @@ -100,21 +101,21 @@ - + -