From fe2dceb66d056809d9a145773a8053ac9fb02658 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 15:22:02 +0000 Subject: [PATCH 01/34] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/utils.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 3ff9f9452..51e74ac8f 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -1,8 +1,9 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPI.Utils do + require Logger alias Calendar.Strftime alias Comeonin.Pbkdf2 alias Pleroma.{Activity, Formatter, Object, Repo} @@ -32,9 +33,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids) do - Enum.map(ids || [], fn media_id -> + def attachments_from_ids(ids, descs) do + Enum.map(ids || [], fn media_id -> do + Logger.warn(descs[media_id]) Repo.get(Object, media_id).data + end end) end From 4c95545d194e8a807e9e3514ed75347d78ec0856 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 15:35:41 +0000 Subject: [PATCH 02/34] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e474653ff..50074b8b0 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Web.CommonAPI do limit = Pleroma.Config.get([:instance, :limit]) with status <- String.trim(status), - attachments <- attachments_from_ids(data["media_ids"]), + attachments <- attachments_from_ids(data["media_ids"], data["descriptions"]), mentions <- Formatter.parse_mentions(status), inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 51e74ac8f..5fe21fb99 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -1,9 +1,8 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPI.Utils do - require Logger alias Calendar.Strftime alias Comeonin.Pbkdf2 alias Pleroma.{Activity, Formatter, Object, Repo} @@ -33,11 +32,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids, descs) do - Enum.map(ids || [], fn media_id -> do - Logger.warn(descs[media_id]) - Repo.get(Object, media_id).data - end + def attachments_from_ids(ids, descs_str) do + {_, descs} = Jason.decode(descs_str) + + Enum.map(ids || [], fn media_id -> + Map.put(Repo.get(Object, media_id).data, "name", descs[media_id]) end) end From ba93396649f65a1f32eeedfd9ccd32cf308e7210 Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Fri, 4 Jan 2019 16:27:46 +0000 Subject: [PATCH 03/34] Patch to support image descriptions in Pleroma FE --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 16 +++++++++++++++- priv/static/index.html | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 50074b8b0..ef79b9c5d 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Web.CommonAPI do limit = Pleroma.Config.get([:instance, :limit]) with status <- String.trim(status), - attachments <- attachments_from_ids(data["media_ids"], data["descriptions"]), + attachments <- attachments_from_ids(data), mentions <- Formatter.parse_mentions(status), inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 5fe21fb99..59df48ed6 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -32,7 +32,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(ids, descs_str) do + def attachments_from_ids(data) do + if Map.has_key?(data, "descriptions") do + attachments_from_ids_descs(data["media_ids"], data["descriptions"]) + else + attachments_from_ids_no_descs(data["media_ids"]) + end + end + + def attachments_from_ids_no_descs(ids) do + Enum.map(ids || [], fn media_id -> + Repo.get(Object, media_id).data + end) + end + + def attachments_from_ids_descs(ids, descs_str) do {_, descs} = Jason.decode(descs_str) Enum.map(ids || [], fn media_id -> diff --git a/priv/static/index.html b/priv/static/index.html index 83c11118c..068d2faec 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -1 +1 @@ -Pleroma
\ No newline at end of file +Pleroma
\ No newline at end of file From bf5b1c7f06c9f8882c40cf2385439bee3b74522c Mon Sep 17 00:00:00 2001 From: Wim Vanderbauwhede Date: Sun, 20 Jan 2019 11:19:50 +0000 Subject: [PATCH 04/34] Removed file as requested --- priv/static/index.html | 1 - 1 file changed, 1 deletion(-) delete mode 100644 priv/static/index.html diff --git a/priv/static/index.html b/priv/static/index.html deleted file mode 100644 index 068d2faec..000000000 --- a/priv/static/index.html +++ /dev/null @@ -1 +0,0 @@ -Pleroma
\ No newline at end of file From 55affbca7fcb214c71b3f8378b0de869c4d4d072 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 28 Jan 2019 22:17:17 +0700 Subject: [PATCH 05/34] add a job queue --- config/config.exs | 6 +- config/test.exs | 2 + docs/config.md | 31 ++++- lib/pleroma/application.ex | 5 +- lib/pleroma/jobs.ex | 153 +++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++- .../web/activity_pub/activity_pub_controller.ex | 4 +- lib/pleroma/web/activity_pub/utils.ex | 2 +- lib/pleroma/web/federator/federator.ex | 139 +++++++------------ lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/websub/websub.ex | 8 +- lib/pleroma/web/websub/websub_controller.ex | 2 +- test/jobs_test.exs | 83 +++++++++++ test/support/jobs_worker_mock.ex | 19 +++ test/web/federator_test.exs | 24 +--- 15 files changed, 358 insertions(+), 142 deletions(-) create mode 100644 lib/pleroma/jobs.ex create mode 100644 test/jobs_test.exs create mode 100644 test/support/jobs_worker_mock.ex diff --git a/config/config.exs b/config/config.exs index e8cf2ed3a..98dd8eb65 100644 --- a/config/config.exs +++ b/config/config.exs @@ -271,14 +271,16 @@ config :pleroma, Pleroma.User, "web" ] -config :pleroma, Pleroma.Web.Federator, max_jobs: 50 - config :pleroma, Pleroma.Web.Federator.RetryQueue, enabled: false, max_jobs: 20, initial_timeout: 30, max_retries: 5 +config :pleroma, Pleroma.Jobs, + federator_incoming: [max_jobs: 50], + federator_outgoing: [max_jobs: 50] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 67ed4737f..7e833fbb3 100644 --- a/config/test.exs +++ b/config/test.exs @@ -43,6 +43,8 @@ config :web_push_encryption, :vapid_details, "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" +config :pleroma, Pleroma.Jobs, testing: [max_jobs: 2] + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index 5464fa90d..84a1e3607 100644 --- a/docs/config.md +++ b/docs/config.md @@ -36,14 +36,15 @@ This filter replaces the filename (not the path) of an upload. For complete obfu An example for Sendgrid adapter: -``` +```exs config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendgrid, api_key: "YOUR_API_KEY" ``` An example for SMTP adapter: -``` + +```exs config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.SMTP, relay: "smtp.gmail.com", @@ -163,7 +164,7 @@ their ActivityPub ID. An example: -``` +```exs config :pleroma, :mrf_user_allowlist, "example.org": ["https://example.org/users/admin"] ``` @@ -192,18 +193,34 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: -``` +```exs config :pleroma, :admin_token, "somerandomtoken" ``` You can then do -``` + +```sh curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" ``` -## Pleroma.Web.Federator +## Pleroma.Jobs + +A list of job queues and their settings. + +Job queue settings: + +* `max_jobs`: The maximum amount of parallel jobs running at the same time. + +Example: + +```exs +config :pleroma, Pleroma.Jobs, + federator_incoming: [max_jobs: 50], + federator_outgoing: [max_jobs: 50] +``` + +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. ## Pleroma.Web.Federator.RetryQueue diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 47c0e5b68..60cba1580 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -101,9 +101,10 @@ defmodule Pleroma.Application do ), worker(Pleroma.FlakeId, []), worker(Pleroma.Web.Federator.RetryQueue, []), - worker(Pleroma.Web.Federator, []), worker(Pleroma.Stats, []), - worker(Pleroma.Web.Push, []) + worker(Pleroma.Web.Push, []), + worker(Pleroma.Jobs, []), + worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) ] ++ streamer_child() ++ chat_child() ++ diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex new file mode 100644 index 000000000..dff0f2197 --- /dev/null +++ b/lib/pleroma/jobs.ex @@ -0,0 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Jobs do + @moduledoc """ + A basic job queue + """ + use GenServer + + require Logger + + def init(args) do + {:ok, args} + end + + def start_link do + queues = + Pleroma.Config.get(Pleroma.Jobs) + |> Enum.map(fn {name, _} -> create_queue(name) end) + |> Enum.into(%{}) + + state = %{ + queues: queues, + refs: %{} + } + + GenServer.start_link(__MODULE__, state, name: __MODULE__) + end + + def create_queue(name) do + {name, {:sets.new(), []}} + end + + @doc """ + Enqueues a job. + + Returns `:ok`. + + ## Arguments + + - `queue_name` - a queue name(must be specified in the config). + - `mod` - a worker module, must have `perform` function. + - `args` - a list of arguments for the `perform` function of the worker module. + - `priority` - a job priority (`0` by default). + + ## Examples + + Enqueue `Module.perform/0` with `priority=1`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, []) + :ok + + Enqueue `Module.perform(:job_name)` with `priority=5`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:job_name], 5) + :ok + + Enqueue `Module.perform(:another_job, data)` with `priority=1`: + + iex> data = "foobar" + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:another_job, data]) + :ok + + Enqueue `Module.perform(:foobar_job, :foo, :bar, 42)` with `priority=1`: + + iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:foobar_job, :foo, :bar, 42]) + :ok + + """ + + def enqueue(queue_name, mod, args, priority \\ 1) + + if Mix.env() == :test do + def enqueue(_queue_name, mod, args, _priority) do + apply(mod, :perform, args) + end + else + @spec enqueue(atom(), atom(), [any()], integer()) :: :ok + def enqueue(queue_name, mod, args, priority \\ 1) do + GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) + end + end + + def handle_cast({:enqueue, queue_name, mod, args, priority}, state) do + {running_jobs, queue} = state[:queues][queue_name] + + queue = enqueue_sorted(queue, {mod, args}, priority) + + state = + state + |> update_queue(queue_name, {running_jobs, queue}) + |> maybe_start_job(queue_name, running_jobs, queue) + + {:noreply, state} + end + + def handle_cast(m, state) do + IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") + {:noreply, state} + end + + def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do + queue_name = state.refs[ref] + + {running_jobs, queue} = state[:queues][queue_name] + + running_jobs = :sets.del_element(ref, running_jobs) + + state = state |> remove_ref(ref) |> maybe_start_job(queue_name, running_jobs, queue) + + {:noreply, state} + end + + def maybe_start_job(state, queue_name, running_jobs, queue) do + if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, queue_name, :max_jobs]) && + queue != [] do + {{mod, args}, queue} = queue_pop(queue) + {:ok, pid} = Task.start(fn -> apply(mod, :perform, args) end) + mref = Process.monitor(pid) + + state + |> add_ref(queue_name, mref) + |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue}) + else + update_queue(state, queue_name, {running_jobs, queue}) + end + end + + def enqueue_sorted(queue, element, priority) do + [%{item: element, priority: priority} | queue] + |> Enum.sort_by(fn %{priority: priority} -> priority end) + end + + def queue_pop([%{item: element} | queue]) do + {element, queue} + end + + defp add_ref(state, queue_name, ref) do + refs = Map.put(state[:refs], ref, queue_name) + Map.put(state, :refs, refs) + end + + defp remove_ref(state, ref) do + refs = Map.delete(state[:refs], ref) + Map.put(state, :refs, refs) + end + + defp update_queue(state, queue_name, data) do + queues = Map.put(state[:queues], queue_name, data) + Map.put(state, :queues, queues) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6b4682e35..5be359887 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -711,20 +711,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = is_public?(activity) - remote_inboxes = - (Pleroma.Web.Salmon.remote_users(activity) ++ followers) - |> Enum.filter(fn user -> User.ap_enabled?(user) end) - |> Enum.map(fn %{info: %{source_data: data}} -> - (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] - end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - Enum.each(remote_inboxes, fn inbox -> - Federator.enqueue(:publish_single_ap, %{ + (Pleroma.Web.Salmon.remote_users(activity) ++ followers) + |> Enum.filter(fn user -> User.ap_enabled?(user) end) + |> Enum.map(fn %{info: %{source_data: data}} -> + (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] + end) + |> Enum.uniq() + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Enum.each(fn inbox -> + Federator.publish_single_ap(%{ inbox: inbox, json: json, actor: actor, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7eed0a600..04c6fef3f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -150,13 +150,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do with %User{} = user <- User.get_cached_by_nickname(nickname), true <- Utils.recipient_in_message(user.ap_id, params), params <- Utils.maybe_splice_recipient(user.ap_id, params) do - Federator.enqueue(:incoming_ap_doc, params) + Federator.incoming_ap_doc(params) json(conn, "ok") end end def inbox(%{assigns: %{valid_signature: true}} = conn, params) do - Federator.enqueue(:incoming_ap_doc, params) + Federator.incoming_ap_doc(params) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e40d05fcd..2ff8a1e8a 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do _ -> 5 end - Pleroma.Web.Federator.enqueue(:publish, activity, priority) + Pleroma.Web.Federator.publish(activity, priority) :ok end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f3a0e18b8..4d03b4622 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do - use GenServer alias Pleroma.User alias Pleroma.Activity + alias Pleroma.Jobs alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub @@ -18,39 +18,60 @@ defmodule Pleroma.Web.Federator do @websub Application.get_env(:pleroma, :websub) @ostatus Application.get_env(:pleroma, :ostatus) - def init(args) do - {:ok, args} + def init() do + # 1 minute + Process.sleep(1000 * 60 * 1) + refresh_subscriptions() end - def start_link do - spawn(fn -> - # 1 minute - Process.sleep(1000 * 60 * 1) - enqueue(:refresh_subscriptions, nil) - end) + # Client API - GenServer.start_link( - __MODULE__, - %{ - in: {:sets.new(), []}, - out: {:sets.new(), []} - }, - name: __MODULE__ - ) + def incoming_doc(doc) do + Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc]) end - def handle(:refresh_subscriptions, _) do + def incoming_ap_doc(params) do + Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params]) + end + + def publish(activity, priority \\ 1) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish, activity], priority) + end + + def publish_single_ap(params) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_ap, params]) + end + + def publish_single_websub(websub) do + Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_websub, websub]) + end + + def verify_websub(websub) do + Jobs.enqueue(:federator_out, __MODULE__, [:verify_websub, websub]) + end + + def request_subscription(sub) do + Jobs.enqueue(:federator_out, __MODULE__, [:request_subscription, sub]) + end + + def refresh_subscriptions() do + Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions]) + end + + # Job Worker Callbacks + + def perform(:refresh_subscriptions) do Logger.debug("Federator running refresh subscriptions") Websub.refresh_subscriptions() spawn(fn -> # 6 hours Process.sleep(1000 * 60 * 60 * 6) - enqueue(:refresh_subscriptions, nil) + refresh_subscriptions() end) end - def handle(:request_subscription, websub) do + def perform(:request_subscription, websub) do Logger.debug("Refreshing #{websub.topic}") with {:ok, websub} <- Websub.request_subscription(websub) do @@ -60,7 +81,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:publish, activity) do + def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do @@ -86,7 +107,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:verify_websub, websub) do + def perform(:verify_websub, websub) do Logger.debug(fn -> "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" end) @@ -94,12 +115,12 @@ defmodule Pleroma.Web.Federator do @websub.verify(websub) end - def handle(:incoming_doc, doc) do + def perform(:incoming_doc, doc) do Logger.info("Got document, trying to parse") @ostatus.handle_incoming(doc) end - def handle(:incoming_ap_doc, params) do + def perform(:incoming_ap_doc, params) do Logger.info("Handling incoming AP activity") params = Utils.normalize_params(params) @@ -124,7 +145,7 @@ defmodule Pleroma.Web.Federator do end end - def handle(:publish_single_ap, params) do + def perform(:publish_single_ap, params) do case ActivityPub.publish_one(params) do {:ok, _} -> :ok @@ -134,7 +155,7 @@ defmodule Pleroma.Web.Federator do end end - def handle( + def perform( :publish_single_websub, %{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params ) do @@ -147,75 +168,11 @@ defmodule Pleroma.Web.Federator do end end - def handle(type, _) do + def perform(type, _) do Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what to do with this"} end - if Mix.env() == :test do - def enqueue(type, payload, _priority \\ 1) do - if Pleroma.Config.get([:instance, :federating]) do - handle(type, payload) - end - end - else - def enqueue(type, payload, priority \\ 1) do - if Pleroma.Config.get([:instance, :federating]) do - GenServer.cast(__MODULE__, {:enqueue, type, payload, priority}) - end - end - end - - def maybe_start_job(running_jobs, queue) do - if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do - {{type, payload}, queue} = queue_pop(queue) - {:ok, pid} = Task.start(fn -> handle(type, payload) end) - mref = Process.monitor(pid) - {:sets.add_element(mref, running_jobs), queue} - else - {running_jobs, queue} - end - end - - def handle_cast({:enqueue, type, payload, _priority}, state) - when type in [:incoming_doc, :incoming_ap_doc] do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_queue = enqueue_sorted(i_queue, {type, payload}, 1) - {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def handle_cast({:enqueue, type, payload, _priority}, state) do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - o_queue = enqueue_sorted(o_queue, {type, payload}, 1) - {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def handle_cast(m, state) do - IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do - %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_running_jobs = :sets.del_element(ref, i_running_jobs) - o_running_jobs = :sets.del_element(ref, o_running_jobs) - {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) - {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) - - {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} - end - - def enqueue_sorted(queue, element, priority) do - [%{item: element, priority: priority} | queue] - |> Enum.sort_by(fn %{priority: priority} -> priority end) - end - - def queue_pop([%{item: element} | queue]) do - {element, queue} - end - def ap_enabled_actor(id) do user = User.get_by_ap_id(id) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 823619edb..845bc60bb 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do {:ok, body, _conn} = read_body(conn) {:ok, doc} = decode_or_retry(body) - Federator.enqueue(:incoming_doc, doc) + Federator.incoming_doc(doc) conn |> send_resp(200, "") diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ca62c83b..652ffd92c 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.{XML, Endpoint, OStatus} + alias Pleroma.Web.{XML, Endpoint, OStatus, Federator} alias Pleroma.Web.Router.Helpers require Logger @@ -77,7 +77,7 @@ defmodule Pleroma.Web.Websub do secret: sub.secret } - Pleroma.Web.Federator.enqueue(:publish_single_websub, data) + Federator.publish_single_websub(data) end) end @@ -109,7 +109,7 @@ defmodule Pleroma.Web.Websub do websub = Repo.update!(change) - Pleroma.Web.Federator.enqueue(:verify_websub, websub) + Federator.verify_websub(websub) {:ok, websub} else @@ -259,7 +259,7 @@ defmodule Pleroma.Web.Websub do subs = Repo.all(query) Enum.each(subs, fn sub -> - Pleroma.Web.Federator.enqueue(:request_subscription, sub) + Federator.request_subscription(sub) end) end diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e58f144e5..eb10227cb 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -80,7 +80,7 @@ defmodule Pleroma.Web.Websub.WebsubController do %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do - Federator.enqueue(:incoming_doc, body) + Federator.incoming_doc(body) conn |> send_resp(200, "OK") diff --git a/test/jobs_test.exs b/test/jobs_test.exs new file mode 100644 index 000000000..ccb518dec --- /dev/null +++ b/test/jobs_test.exs @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobsTest do + use ExUnit.Case, async: true + + alias Pleroma.Jobs + alias Jobs.WorkerMock + + setup do + state = %{ + queues: Enum.into([Jobs.create_queue(:testing)], %{}), + refs: %{} + } + + [state: state] + end + + test "creates queue" do + queue = Jobs.create_queue(:foobar) + + assert {:foobar, set} = queue + assert :set == elem(set, 0) |> elem(0) + end + + test "enqueues an element according to priority" do + queue = [%{item: 1, priority: 2}] + + new_queue = Jobs.enqueue_sorted(queue, 2, 1) + assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + new_queue = Jobs.enqueue_sorted(queue, 2, 3) + assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] + end + + test "pop first item" do + queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + assert {2, [%{item: 1, priority: 2}]} = Jobs.queue_pop(queue) + end + + test "enqueue a job", %{state: state} do + assert {:noreply, new_state} = + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + + assert %{queues: %{testing: {running_jobs, []}}, refs: _} = new_state + assert :sets.size(running_jobs) == 1 + assert [ref] = :sets.to_list(running_jobs) + assert %{refs: %{^ref => :testing}} = new_state + end + + test "max jobs setting", %{state: state} do + max_jobs = Pleroma.Config.get([Jobs, :testing, :max_jobs]) + + {:noreply, state} = + Enum.reduce(1..(max_jobs + 1), {:noreply, state}, fn _, {:noreply, state} -> + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + end) + + assert %{ + queues: %{ + testing: + {running_jobs, [%{item: {WorkerMock, [:test_job, :foo, :bar]}, priority: 3}]} + } + } = state + + assert :sets.size(running_jobs) == max_jobs + end + + test "remove job after it finished", %{state: state} do + {:noreply, new_state} = + Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state) + + %{queues: %{testing: {running_jobs, []}}} = new_state + [ref] = :sets.to_list(running_jobs) + + assert {:noreply, %{queues: %{testing: {running_jobs, []}}, refs: %{}}} = + Jobs.handle_info({:DOWN, ref, :process, nil, nil}, new_state) + + assert :sets.size(running_jobs) == 0 + end +end diff --git a/test/support/jobs_worker_mock.ex b/test/support/jobs_worker_mock.ex new file mode 100644 index 000000000..0fb976d05 --- /dev/null +++ b/test/support/jobs_worker_mock.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Jobs.WorkerMock do + require Logger + + def perform(:test_job, arg, arg2) do + Logger.debug({:perform, :test_job, arg, arg2}) + end + + def perform(:test_job, payload) do + Logger.debug({:perform, :test_job, payload}) + end + + def test_job(payload) do + Pleroma.Jobs.enqueue(:testing, __MODULE__, [:test_job, payload]) + end +end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index a49265c0c..d58621cc3 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -14,22 +14,6 @@ defmodule Pleroma.Web.FederatorTest do :ok end - test "enqueues an element according to priority" do - queue = [%{item: 1, priority: 2}] - - new_queue = Federator.enqueue_sorted(queue, 2, 1) - assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] - - new_queue = Federator.enqueue_sorted(queue, 2, 3) - assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] - end - - test "pop first item" do - queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] - - assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue) - end - describe "Publish an activity" do setup do user = insert(:user) @@ -49,7 +33,7 @@ defmodule Pleroma.Web.FederatorTest do relay_mock: relay_mock } do with_mocks([relay_mock]) do - Federator.handle(:publish, activity) + Federator.publish(activity) end assert_received :relay_publish @@ -62,7 +46,7 @@ defmodule Pleroma.Web.FederatorTest do Pleroma.Config.put([:instance, :allow_relay], false) with_mocks([relay_mock]) do - Federator.handle(:publish, activity) + Federator.publish(activity) end refute_received :relay_publish @@ -87,7 +71,7 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - {:ok, _activity} = Federator.handle(:incoming_ap_doc, params) + {:ok, _activity} = Federator.incoming_ap_doc(params) end test "rejects incoming AP docs with incorrect origin" do @@ -105,7 +89,7 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - :error = Federator.handle(:incoming_ap_doc, params) + :error = Federator.incoming_ap_doc(params) end end end From b2e9700785124a8ba4cfb037ae92a6bfc7c3b224 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 28 Jan 2019 23:12:02 +0700 Subject: [PATCH 06/34] cleanup --- lib/pleroma/jobs.ex | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index dff0f2197..2a75ff529 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Jobs do ## Arguments - `queue_name` - a queue name(must be specified in the config). - - `mod` - a worker module, must have `perform` function. + - `mod` - a worker module (must have `perform` function). - `args` - a list of arguments for the `perform` function of the worker module. - `priority` - a job priority (`0` by default). @@ -76,7 +76,6 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else - @spec enqueue(atom(), atom(), [any()], integer()) :: :ok def enqueue(queue_name, mod, args, priority \\ 1) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end @@ -95,11 +94,6 @@ defmodule Pleroma.Jobs do {:noreply, state} end - def handle_cast(m, state) do - IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do queue_name = state.refs[ref] From 56533495b53c12223678de3f2fc6df8ee1165c19 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 29 Jan 2019 00:33:08 +0700 Subject: [PATCH 07/34] fix defaults --- lib/pleroma/jobs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index 2a75ff529..d42688193 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else - def enqueue(queue_name, mod, args, priority \\ 1) do + def enqueue(queue_name, mod, args, priority) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end end From 1724a6b34b099dc83b94687d27d2492aaf97013e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 29 Jan 2019 00:35:57 +0700 Subject: [PATCH 08/34] add spec back --- lib/pleroma/jobs.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index d42688193..16dccb682 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -76,6 +76,7 @@ defmodule Pleroma.Jobs do apply(mod, :perform, args) end else + @spec enqueue(atom(), atom(), [any()], integer()) :: :ok def enqueue(queue_name, mod, args, priority) do GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority}) end From ab31adf15bbec1597a9b7cf065898fb3f712eef3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 30 Jan 2019 22:56:59 +0700 Subject: [PATCH 09/34] tiny improve --- lib/pleroma/jobs.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/jobs.ex b/lib/pleroma/jobs.ex index 16dccb682..24b7e5e46 100644 --- a/lib/pleroma/jobs.ex +++ b/lib/pleroma/jobs.ex @@ -102,7 +102,11 @@ defmodule Pleroma.Jobs do running_jobs = :sets.del_element(ref, running_jobs) - state = state |> remove_ref(ref) |> maybe_start_job(queue_name, running_jobs, queue) + state = + state + |> remove_ref(ref) + |> update_queue(queue_name, {running_jobs, queue}) + |> maybe_start_job(queue_name, running_jobs, queue) {:noreply, state} end @@ -118,7 +122,7 @@ defmodule Pleroma.Jobs do |> add_ref(queue_name, mref) |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue}) else - update_queue(state, queue_name, {running_jobs, queue}) + state end end From 58e250d9d27205116df5634d909ec1e43ea55286 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 31 Jan 2019 15:23:50 +0700 Subject: [PATCH 10/34] fix merge --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e81816e35..e44c48c35 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,7 +109,7 @@ defmodule Pleroma.Application do worker(Pleroma.Stats, []), worker(Pleroma.Web.Push, []), worker(Pleroma.Jobs, []), - worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) + worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) ] ++ streamer_child() ++ chat_child() ++ From 5b1d7c3c5672af065af503891d156b6e0cf5a8c1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 6 Feb 2019 12:17:41 +0700 Subject: [PATCH 11/34] fix tests --- test/web/federator_test.exs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 147086918..b56694a8b 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -87,11 +87,9 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Federator.enqueue(:publish_single_ap, %{inbox: inbox1, unreachable_since: dt}) - ) + assert called(Federator.publish_single_ap(%{inbox: inbox1, unreachable_since: dt})) - refute called(Federator.enqueue(:publish_single_ap, %{inbox: inbox2})) + refute called(Federator.publish_single_ap(%{inbox: inbox2})) end test_with_mock "it federates only to reachable instances via Websub", @@ -123,13 +121,13 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) assert called( - Federator.enqueue(:publish_single_websub, %{ + Federator.publish_single_websub(%{ callback: sub2.callback, unreachable_since: dt }) ) - refute called(Federator.enqueue(:publish_single_websub, %{callback: sub1.callback})) + refute called(Federator.publish_single_websub(%{callback: sub1.callback})) end test_with_mock "it federates only to reachable instances via Salmon", @@ -163,13 +161,13 @@ defmodule Pleroma.Web.FederatorTest do CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) assert called( - Federator.enqueue(:publish_single_salmon, %{ + Federator.publish_single_salmon(%{ recipient: remote_user2, unreachable_since: dt }) ) - refute called(Federator.enqueue(:publish_single_websub, %{recipient: remote_user1})) + refute called(Federator.publish_single_websub(%{recipient: remote_user1})) end end From 61a4bc50952b11a59dce7f655c883de59306adcd Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 10 Feb 2019 22:41:06 +0300 Subject: [PATCH 12/34] Add OAuth tokens endpoint --- lib/pleroma/web/oauth/token.ex | 8 ++++++++ lib/pleroma/web/router.ex | 2 ++ .../web/twitter_api/twitter_api_controller.ex | 12 ++++++++++++ lib/pleroma/web/twitter_api/views/token_view.ex | 22 ++++++++++++++++++++++ test/support/factory.ex | 13 +++++++++++++ .../twitter_api/twitter_api_controller_test.exs | 18 ++++++++++++++++++ 6 files changed, 75 insertions(+) create mode 100644 lib/pleroma/web/twitter_api/views/token_view.ex diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index b0bbeeb69..40bf0ac6b 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -52,4 +52,12 @@ defmodule Pleroma.Web.OAuth.Token do ) |> Repo.delete_all() end + + def get_user_tokens(%User{id: user_id}) do + from( + t in Pleroma.Web.OAuth.Token, + where: t.user_id == ^user_id + ) + |> Repo.all() + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5b5627ce8..a394900b2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -389,6 +389,8 @@ defmodule Pleroma.Web.Router do get("/qvitter/mutes", TwitterAPI.Controller, :raw_empty_array) get("/externalprofile/show", TwitterAPI.Controller, :external_profile) + + get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) end pipeline :ap_relay do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..1a43e9a60 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -8,6 +8,10 @@ defmodule Pleroma.Web.TwitterAPI.Controller do import Pleroma.Web.ControllerHelper, only: [json_response: 3] alias Ecto.Changeset + alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView, NotificationView, TokenView} + alias Pleroma.Web.CommonAPI + alias Pleroma.{Repo, Activity, Object, User, Notification} + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI @@ -542,6 +546,14 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def oauth_tokens(%{assigns: %{user: user}} = conn, _params) do + with oauth_tokens <- Token.get_user_tokens(user) do + conn + |> put_view(TokenView) + |> render("index.json", %{tokens: oauth_tokens}) + end + end + def blocks(%{assigns: %{user: user}} = conn, _params) do with blocked_users <- User.blocked_users(user) do conn diff --git a/lib/pleroma/web/twitter_api/views/token_view.ex b/lib/pleroma/web/twitter_api/views/token_view.ex new file mode 100644 index 000000000..96b8526a4 --- /dev/null +++ b/lib/pleroma/web/twitter_api/views/token_view.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.TokenView do + use Pleroma.Web, :view + + def render("index.json", %{tokens: tokens}) do + tokens + |> render_many(Pleroma.Web.TwitterAPI.TokenView, "show.json") + |> Enum.filter(&Enum.any?/1) + end + + def render("show.json", %{token: token_entry}) do + %{ + id: token_entry.id, + token: token_entry.token, + refresh_token: token_entry.refresh_token, + valid_until: token_entry.valid_until + } + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 0c21093ce..7a91549f5 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -227,4 +227,17 @@ defmodule Pleroma.Factory do unreachable_since: nil } end + + def oauth_token_factory do + user = insert(:user) + oauth_app = insert(:oauth_app) + + %Pleroma.Web.OAuth.Token{ + token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(), + refresh_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(), + user_id: user.id, + app_id: oauth_app.id, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10) + } + end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 855ae1526..c50d82def 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1876,4 +1876,22 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do ActivityRepresenter.to_map(activity, %{user: user, for: user}) end end + + describe "GET /api/oauth_tokens" do + test "renders list" do + token = insert(:oauth_token) + + response = + build_conn() + |> assign(:user, Repo.get(User, token.user_id)) + |> get("/api/oauth_tokens") + + keys = + json_response(response, 200) + |> hd() + |> Map.keys() + + assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] + end + end end From 62a45bdc11bc98ca4c24b0b8aa54c9d2958f81a1 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Feb 2019 00:49:56 +0300 Subject: [PATCH 13/34] Add revoke token --- lib/pleroma/web/oauth/token.ex | 11 ++++++++++- lib/pleroma/web/router.ex | 1 + .../web/twitter_api/twitter_api_controller.ex | 6 ++++++ .../twitter_api/twitter_api_controller_test.exs | 23 +++++++++++++++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 40bf0ac6b..380a39360 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -53,9 +53,18 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.delete_all() end - def get_user_tokens(%User{id: user_id}) do + def delete_user_token(%User{id: user_id}, token_id) do from( t in Pleroma.Web.OAuth.Token, + where: t.user_id == ^user_id, + where: t.id == ^token_id + ) + |> Repo.delete_all() + end + + def get_user_tokens(%User{id: user_id}) do + from( + t in Token, where: t.user_id == ^user_id ) |> Repo.all() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a394900b2..d45fa526e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -391,6 +391,7 @@ defmodule Pleroma.Web.Router do get("/externalprofile/show", TwitterAPI.Controller, :external_profile) get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) + delete("/oauth_tokens/:id", TwitterAPI.Controller, :revoke_token) end pipeline :ap_relay do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 1a43e9a60..fac05f288 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -554,6 +554,12 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def revoke_token(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do + Token.delete_user_token(user, id) + + json_reply(conn, 201, "") + end + def blocks(%{assigns: %{user: user}} = conn, _params) do with blocked_users <- User.blocked_users(user) do conn diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index c50d82def..527a920fb 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Builders.{ActivityBuilder, UserBuilder} alias Pleroma.{Repo, Activity, User, Object, Notification} alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.UserView alias Pleroma.Web.TwitterAPI.NotificationView alias Pleroma.Web.CommonAPI @@ -1878,12 +1879,16 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do end describe "GET /api/oauth_tokens" do - test "renders list" do - token = insert(:oauth_token) + setup do + token = insert(:oauth_token) |> Repo.preload(:user) + %{token: token} + end + + test "renders list", %{token: token} do response = build_conn() - |> assign(:user, Repo.get(User, token.user_id)) + |> assign(:user, token.user) |> get("/api/oauth_tokens") keys = @@ -1893,5 +1898,17 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] end + + test "revoke token", %{token: token} do + response = + build_conn() + |> assign(:user, token.user) + |> delete("/api/oauth_tokens/#{token.id}") + + tokens = Token.get_user_tokens(token.user) + + assert tokens == [] + assert response.status == 201 + end end end From 760fec4cb85c7ddf2ec4fa5578ab4a1ceafc1e84 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Feb 2019 09:48:24 +0000 Subject: [PATCH 14/34] Update token.ex --- lib/pleroma/web/oauth/token.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 380a39360..f5594f834 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Web.OAuth.Token do def delete_user_tokens(%User{id: user_id}) do from( - t in Pleroma.Web.OAuth.Token, + t in Token, where: t.user_id == ^user_id ) |> Repo.delete_all() @@ -55,7 +55,7 @@ defmodule Pleroma.Web.OAuth.Token do def delete_user_token(%User{id: user_id}, token_id) do from( - t in Pleroma.Web.OAuth.Token, + t in Token, where: t.user_id == ^user_id, where: t.id == ^token_id ) From 3f32d7b937a2368707794b55d1256bfa9fa508f4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 14 Feb 2019 17:02:47 +0700 Subject: [PATCH 15/34] Fix queue name --- lib/pleroma/web/federator/federator.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 7df75aca6..d4e2a9742 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -38,31 +38,31 @@ defmodule Pleroma.Web.Federator do end def publish(activity, priority \\ 1) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish, activity], priority) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority) end def publish_single_ap(params) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_ap, params]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params]) end def publish_single_websub(websub) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_websub, websub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub]) end def verify_websub(websub) do - Jobs.enqueue(:federator_out, __MODULE__, [:verify_websub, websub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub]) end def request_subscription(sub) do - Jobs.enqueue(:federator_out, __MODULE__, [:request_subscription, sub]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub]) end def refresh_subscriptions() do - Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions]) end def publish_single_salmon(params) do - Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_salmon, params]) + Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params]) end # Job Worker Callbacks From ecdf0657ba4a90d821d3874c827593963e0ff041 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sun, 10 Feb 2019 02:26:29 +0300 Subject: [PATCH 16/34] Add logic for keeping follow_request_count up-to-date on the `follow`, `approve_friend_request`, and `deny_friend_request` actions. Add follow_request_count to the user view. --- lib/pleroma/user.ex | 26 ++++++++++++++++++++++ lib/pleroma/user/info.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 16 ++++++++----- lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++--- .../web/mastodon_api/mastodon_api_controller.ex | 4 ++-- .../web/twitter_api/twitter_api_controller.ex | 4 ++-- lib/pleroma/web/twitter_api/views/user_view.ex | 18 +++++++++++---- .../mastodon_api/mastodon_api_controller_test.exs | 8 ++++++- .../twitter_api/twitter_api_controller_test.exs | 26 ++++++++++++++++++++++ 9 files changed, 91 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3232cb842..854787a2b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -618,6 +618,32 @@ defmodule Pleroma.User do ) end + def update_follow_request_count(%User{} = user) do + subquery = + user + |> User.get_follow_requests_query() + |> select([a], %{count: count(a.id)}) + + User + |> where(id: ^user.id) + |> join(:inner, [u], s in subquery(subquery)) + |> update([u, s], + set: [ + info: + fragment( + "jsonb_set(?, '{follow_request_count}', ?::varchar::jsonb, true)", + u.info, + s.count + ) + ] + ) + |> Repo.update_all([], returning: true) + |> case do + {1, [user]} -> {:ok, user} + _ -> {:error, user} + end + end + def get_follow_requests(%User{} = user) do q = get_follow_requests_query(user) reqs = Repo.all(q) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9d8779fab..c59e74c45 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -12,6 +12,7 @@ defmodule Pleroma.User.Info do field(:source_data, :map, default: %{}) field(:note_count, :integer, default: 0) field(:follower_count, :integer, default: 0) + field(:follow_request_count, :integer, default: 0) field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:confirmation_token, :string, default: nil) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c46d8233e..975f9fde3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -172,9 +172,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) - with data <- %{"to" => to, "type" => "Accept", "actor" => actor, "object" => object}, + with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object}, {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(actor) do {:ok, activity} end end @@ -183,9 +184,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) - with data <- %{"to" => to, "type" => "Reject", "actor" => actor, "object" => object}, + with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object}, {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(actor) do {:ok, activity} end end @@ -283,7 +285,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def follow(follower, followed, activity_id \\ nil, local \\ true) do with data <- make_follow_data(follower, followed, activity_id), {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(followed) do {:ok, activity} end end @@ -293,7 +296,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, follow_activity} <- update_follow_state(follow_activity, "cancelled"), unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id), {:ok, activity} <- insert(unfollow_data, local), - :ok <- maybe_federate(activity) do + :ok <- maybe_federate(activity), + _ <- User.update_follow_request_count(followed) do {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 26b2dd575..41d89a02b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -406,7 +406,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do if not User.locked?(followed) do ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: data, local: true }) @@ -432,7 +432,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ActivityPub.accept(%{ to: follow_activity.data["to"], type: "Accept", - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], local: false }) do @@ -458,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ActivityPub.reject(%{ to: follow_activity.data["to"], type: "Reject", - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], local: false }) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index dcaeccac6..f0bbe5b3f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -680,7 +680,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, _activity} <- ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Accept" }) do @@ -702,7 +702,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, _activity} <- ActivityPub.reject(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Reject" }) do diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..70ae4068a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -570,7 +570,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do {:ok, _activity} <- ActivityPub.accept(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Accept" }) do @@ -590,7 +590,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do {:ok, _activity} <- ActivityPub.reject(%{ to: [follower.ap_id], - actor: followed.ap_id, + actor: followed, object: follow_activity.data["id"], type: "Reject" }) do diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index a09450df7..df7384476 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -113,10 +113,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "fields" => fields, # Pleroma extension - "pleroma" => %{ - "confirmation_pending" => user_info.confirmation_pending, - "tags" => user.tags - } + "pleroma" => + %{ + "confirmation_pending" => user_info.confirmation_pending, + "tags" => user.tags + } + |> maybe_with_follow_request_count(user, for_user) } data = @@ -132,6 +134,14 @@ defmodule Pleroma.Web.TwitterAPI.UserView do end end + defp maybe_with_follow_request_count(data, %User{id: id, info: %{locked: true}} = user, %User{ + id: id + }) do + Map.put(data, "follow_request_count", user.info.follow_request_count) + end + + defp maybe_with_follow_request_count(data, _, _), do: data + defp maybe_with_role(data, %User{id: id} = user, %User{id: id}) do Map.merge(data, %{"role" => role(user), "show_role" => user.info.show_role}) end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 26c9c25a6..7749c5ded 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -937,7 +937,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "/api/v1/follow_requests/:id/authorize works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) @@ -946,6 +946,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() @@ -959,6 +960,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == true + assert user.info.follow_request_count == 0 end test "verify_credentials", %{conn: conn} do @@ -979,6 +981,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do {:ok, _activity} = ActivityPub.follow(other_user, user) + user = Repo.get(User, user.id) + assert user.info.follow_request_count == 1 + conn = build_conn() |> assign(:user, user) @@ -991,6 +996,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 0 end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index acb03b146..50b19fd86 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -640,6 +640,24 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert json_response(conn, 200) == UserView.render("show.json", %{user: followed, for: current_user}) end + + test "for restricted account", %{conn: conn, user: current_user} do + followed = insert(:user, info: %User.Info{locked: true}) + + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/friendships/create.json", %{user_id: followed.id}) + + current_user = Repo.get(User, current_user.id) + followed = Repo.get(User, followed.id) + + refute User.ap_followers(followed) in current_user.following + assert followed.info.follow_request_count == 1 + + assert json_response(conn, 200) == + UserView.render("show.json", %{user: followed, for: current_user}) + end end describe "POST /friendships/destroy.json" do @@ -1676,15 +1694,19 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() |> assign(:user, user) |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id}) + user = Repo.get(User, user.id) + assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] assert relationship["follows_you"] == true + assert user.info.follow_request_count == 0 end end @@ -1699,15 +1721,19 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do other_user = Repo.get(User, other_user.id) assert User.following?(other_user, user) == false + assert user.info.follow_request_count == 1 conn = build_conn() |> assign(:user, user) |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id}) + user = Repo.get(User, user.id) + assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] assert relationship["follows_you"] == false + assert user.info.follow_request_count == 0 end end From 38e15930cb7e8aec4742eb85da26955b4c08e8ce Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 16 Feb 2019 17:01:15 +0100 Subject: [PATCH 17/34] Add option to return all friends in twitter api. Mainly useful for user export. --- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 3 +++ test/web/twitter_api/twitter_api_controller_test.exs | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index c2f0dc2a9..a8ef0a8ca 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -524,6 +524,9 @@ defmodule Pleroma.Web.TwitterAPI.Controller do def friends(%{assigns: %{user: for_user}} = conn, params) do {:ok, page} = Ecto.Type.cast(:integer, params["page"] || 1) + {:ok, export} = Ecto.Type.cast(:boolean, params["all"] || false) + + page = if export, do: nil, else: page with {:ok, user} <- TwitterAPI.get_user(conn.assigns[:user], params), {:ok, friends} <- User.get_friends(user, page) do diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index acb03b146..f7e40e0d3 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1218,7 +1218,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert Enum.sort(expected) == Enum.sort(result) end - test "it returns 20 friends per page", %{conn: conn} do + test "it returns 20 friends per page, except if 'export' is set to true", %{conn: conn} do user = insert(:user) followeds = insert_list(21, :user) @@ -1242,6 +1242,14 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do result = json_response(res_conn, 200) assert length(result) == 1 + + res_conn = + conn + |> assign(:user, user) + |> get("/api/statuses/friends", %{all: true}) + + result = json_response(res_conn, 200) + assert length(result) == 21 end test "it returns a given user's friends with user_id", %{conn: conn} do From 96c725328b556833d59de23093fb4aeba9bb5684 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 16 Feb 2019 20:38:25 +0300 Subject: [PATCH 18/34] Remove a limit on attachments in Mastodon API and document the changes in responses from vanilla mastodon --- docs/Differences-in-MastodonAPI-Responses.md | 9 +++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 docs/Differences-in-MastodonAPI-Responses.md diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md new file mode 100644 index 000000000..cfa34593e --- /dev/null +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -0,0 +1,9 @@ +# Differences in Mastodon API responses from vanilla Mastodon + +## Flake IDs + +Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are sortable strings + +## Attachment cap + +Some apps operate under the assumption that no more than 4 attachments ccan be returned, however Pleroma can return any amount of attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 69f5f992c..a49b381c9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -166,7 +166,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do sensitive: sensitive, spoiler_text: object["summary"] || "", visibility: get_visibility(object), - media_attachments: attachments |> Enum.take(4), + media_attachments: attachments, mentions: mentions, tags: build_tags(tags), application: %{ From c788f1543c4a2436c93409423d93284467e431e2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 16 Feb 2019 21:14:07 +0300 Subject: [PATCH 19/34] Add a section on how to identify a pleroma instance, clarify that post upload limit is not capped too --- docs/Differences-in-MastodonAPI-Responses.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md index cfa34593e..488dc9389 100644 --- a/docs/Differences-in-MastodonAPI-Responses.md +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -1,9 +1,11 @@ # Differences in Mastodon API responses from vanilla Mastodon +A Pleroma instance can be identified by " (compatible; Pleroma )" present in `version` field in response from `/api/v1/instance` + ## Flake IDs Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are sortable strings ## Attachment cap -Some apps operate under the assumption that no more than 4 attachments ccan be returned, however Pleroma can return any amount of attachments +Some apps operate under the assumption that no more than 4 attachments ccan be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. From 25ab90edeaae53b6ce084d1ba9a02df5505b5041 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Feb 2019 19:39:03 +0100 Subject: [PATCH 20/34] mix.exs: Add docs/Clients.md to docs.extras [ci skip] --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d46998891..eba3295dd 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,7 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md"], + extras: ["README.md", "docs/Admin-API.md", "docs/Clients.md", "docs/config.md", "docs/Pleroma-API.md"], main: "readme", output: "priv/static/doc" ] From 4df455f69bef5270c7e6a57022237ff75f13687c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 3 Feb 2019 12:31:12 +0100 Subject: [PATCH 21/34] [MastoAPI] Add switching of frontend flavours --- docs/Pleroma-API.md | 14 ++++++++ lib/pleroma/user/info.ex | 9 +++++ .../web/mastodon_api/mastodon_api_controller.ex | 41 +++++++++++++++++++++- lib/pleroma/web/router.ex | 3 ++ .../templates/mastodon_api/mastodon/index.html.eex | 8 ++--- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/docs/Pleroma-API.md b/docs/Pleroma-API.md index e1448d3f0..379d3dbed 100644 --- a/docs/Pleroma-API.md +++ b/docs/Pleroma-API.md @@ -94,3 +94,17 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi ## `/api/pleroma/admin/`… See [Admin-API](Admin-API.md) + +## `/api/v1/pleroma/flavour/:flavour` +* Method `POST` +* Authentication: required +* Response: JSON string. Returns the user flavour or the default one on success, otherwise returns `{"error": "error_msg"}` +* Example response: "glitch" +* Note: This is intended to be used only by mastofe + +## `/api/v1/pleroma/flavour` +* Method `GET` +* Authentication: required +* Response: JSON string. Returns the user flavour or the default one. +* Example response: "glitch" +* Note: This is intended to be used only by mastofe diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9d8779fab..e33ec816b 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -34,6 +34,7 @@ defmodule Pleroma.User.Info do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:pinned_activities, {:array, :string}, default: []) + field(:flavour, :string, default: nil) # Found in the wild # ap_id -> Where is this used? @@ -186,6 +187,14 @@ defmodule Pleroma.User.Info do |> validate_required([:settings]) end + def mastodon_flavour_update(info, flavour) do + params = %{flavour: flavour} + + info + |> cast(params, [:flavour]) + |> validate_required([:flavour]) + end + def set_source_data(info, source_data) do params = %{source_data: source_data} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index dcaeccac6..0150f18f8 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1051,6 +1051,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do accounts = Map.put(%{}, user.id, AccountView.render("account.json", %{user: user, for: user})) + flavour = get_user_flavour(user) + initial_state = %{ meta: %{ @@ -1135,7 +1137,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do conn |> put_layout(false) |> put_view(MastodonView) - |> render("index.html", %{initial_state: initial_state}) + |> render("index.html", %{initial_state: initial_state, flavour: flavour}) else conn |> redirect(to: "/web/login") @@ -1157,6 +1159,43 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + @supported_flavours ["glitch", "vanilla"] + + def set_flavour(%{assigns: %{user: user}} = conn, %{"flavour" => flavour} = _params) + when flavour in @supported_flavours do + flavour_cng = User.Info.mastodon_flavour_update(user.info, flavour) + + with changeset <- Ecto.Changeset.change(user), + changeset <- Ecto.Changeset.put_embed(changeset, :info, flavour_cng), + {:ok, user} <- User.update_and_set_cache(changeset), + flavour <- user.info.flavour do + json(conn, flavour) + else + e -> + conn + |> put_resp_content_type("application/json") + |> send_resp(500, Jason.encode!(%{"error" => inspect(e)})) + end + end + + def set_flavour(conn, _params) do + conn + |> put_status(400) + |> json(%{error: "Unsupported flavour"}) + end + + def get_flavour(%{assigns: %{user: user}} = conn, _params) do + json(conn, get_user_flavour(user)) + end + + defp get_user_flavour(%User{info: %{flavour: flavour}}) when flavour in @supported_flavours do + flavour + end + + defp get_user_flavour(_) do + "glitch" + end + def login(conn, %{"code" => code}) do with {:ok, app} <- get_or_make_app(), %Authorization{} = auth <- Repo.get_by(Authorization, token: code, app_id: app.id), diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d66a1c2a1..664f93c1c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -236,6 +236,9 @@ defmodule Pleroma.Web.Router do get("/suggestions", MastodonAPIController, :suggestions) get("/endorsements", MastodonAPIController, :empty_array) + + post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour) + get("/pleroma/flavour", MastodonAPIController, :get_flavour) end scope "/api/web", Pleroma.Web.MastodonAPI do diff --git a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex index 9a725e420..5659c7828 100644 --- a/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex +++ b/lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex @@ -8,7 +8,7 @@ - + @@ -19,10 +19,10 @@ - - + + - +
From 72a4272d84a68ceb4d9a39ddaa4d3f45779bfebf Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 10 Feb 2019 23:11:12 +0100 Subject: [PATCH 22/34] Web.MastodonAPI.MastodonAPIControllerTest: Add testing of the flavour switching --- .../mastodon_api/mastodon_api_controller_test.exs | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 26c9c25a6..450bf10a3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1786,4 +1786,29 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> json_response(200) end end + + test "flavours switching (Pleroma Extension)", %{conn: conn} do + user = insert(:user) + + get_old_flavour = + conn + |> assign(:user, user) + |> get("/api/v1/pleroma/flavour") + + assert "glitch" == json_response(get_old_flavour, 200) + + set_flavour = + conn + |> assign(:user, user) + |> post("/api/v1/pleroma/flavour/vanilla") + + assert "vanilla" == json_response(set_flavour, 200) + + get_new_flavour = + conn + |> assign(:user, user) + |> post("/api/v1/pleroma/flavour/vanilla") + + assert json_response(set_flavour, 200) == json_response(get_new_flavour, 200) + end end From 006bec8c6a8addb57427937f71ad05ca691fe7b5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 10:34:00 +0300 Subject: [PATCH 23/34] Add differences in MastoAPI responses to mix.exs extras --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d46998891..e06501e5b 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,7 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md"], + extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md", "docs/Differences-in-MastodonAPI-Responses.md"], main: "readme", output: "priv/static/doc" ] From ccd30a187541697dabbcb6449708dc5c4555aabc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 12:07:49 +0300 Subject: [PATCH 24/34] Fix formating --- mix.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index e06501e5b..ed6ca6135 100644 --- a/mix.exs +++ b/mix.exs @@ -21,7 +21,13 @@ defmodule Pleroma.Mixfile do homepage_url: "https://pleroma.social/", docs: [ logo: "priv/static/static/logo.png", - extras: ["README.md", "docs/config.md", "docs/Pleroma-API.md", "docs/Admin-API.md", "docs/Differences-in-MastodonAPI-Responses.md"], + extras: [ + "README.md", + "docs/config.md", + "docs/Pleroma-API.md", + "docs/Admin-API.md", + "docs/Differences-in-MastodonAPI-Responses.md" + ], main: "readme", output: "priv/static/doc" ] From 8f98d970c105bec3205c6ce524750583ad5fb502 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 13:46:40 +0300 Subject: [PATCH 25/34] Fix recipient count in hellthread policy --- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 8ab1dd4e5..6736f3cb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -12,14 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection) message = - case recipients = get_recipient_count(message) do - {:public, _} + case get_recipient_count(message) do + {:public, recipients} when follower_collection? and recipients > threshold -> message |> Map.put("to", [follower_collection]) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) - {:public, _} when recipients > threshold -> + {:public, recipients} when recipients > threshold -> message |> Map.put("to", []) |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"]) From fa191a658b373eb53ab0a44ab0529fb8635cfd74 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 17 Feb 2019 11:36:14 +0100 Subject: [PATCH 26/34] Speed up docker postgres. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b59445895..eeddce69e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,8 @@ image: elixir:1.7.2 services: - - postgres:9.6.2 + - name: postgres:9.6.2 + command: ["postgres", "-c", "fsync=off"] variables: POSTGRES_DB: pleroma_test From d0a94f98e030dd700c56e8cc4576fb45cf449f53 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 Feb 2019 14:33:44 +0300 Subject: [PATCH 27/34] more tests for HellthreadPolicy --- .../activity_pub/mrf/hellthread_policy_test.exs | 65 +++++++++++++++------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs index ebf9997cd..eb6ee4d04 100644 --- a/test/web/activity_pub/mrf/hellthread_policy_test.exs +++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs @@ -8,32 +8,47 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do import Pleroma.Web.ActivityPub.MRF.HellthreadPolicy - describe "hellthread filter tests" do - setup do - user = insert(:user) + setup do + user = insert(:user) - message = %{ - "actor" => user.ap_id, - "cc" => [user.follower_address], - "type" => "Create", - "to" => [ - "https://www.w3.org/ns/activitystreams#Public", - "https://instace.tld/users/user1", - "https://instace.tld/users/user2", - "https://instace.tld/users/user3" - ] - } + message = %{ + "actor" => user.ap_id, + "cc" => [user.follower_address], + "type" => "Create", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public", + "https://instance.tld/users/user1", + "https://instance.tld/users/user2", + "https://instance.tld/users/user3" + ] + } - [user: user, message: message] - end + [user: user, message: message] + end - test "reject test", %{message: message} do + describe "reject" do + test "rejects the message if the recipient count is above reject_threshold", %{ + message: message + } do Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 2}) {:reject, nil} = filter(message) end - test "delist test", %{user: user, message: message} do + test "does not reject the message if the recipient count is below reject_threshold", %{ + message: message + } do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + + assert {:ok, ^message} = filter(message) + end + end + + describe "delist" do + test "delists the message if the recipient count is above delist_threshold", %{ + user: user, + message: message + } do Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 2, reject_threshold: 0}) {:ok, message} = filter(message) @@ -41,10 +56,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do assert "https://www.w3.org/ns/activitystreams#Public" in message["cc"] end - test "excludes follower collection and public URI from threshold count", %{message: message} do - Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + test "does not delist the message if the recipient count is below delist_threshold", %{ + message: message + } do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 4, reject_threshold: 0}) - {:ok, _} = filter(message) + assert {:ok, ^message} = filter(message) end end + + test "excludes follower collection and public URI from threshold count", %{message: message} do + Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 0, reject_threshold: 3}) + + assert {:ok, ^message} = filter(message) + end end From 71c8c60ded8dce402dbe69545afebd55c63927e5 Mon Sep 17 00:00:00 2001 From: lain Date: Sun, 17 Feb 2019 17:47:24 +0100 Subject: [PATCH 28/34] More speedup, test fixes. --- .gitlab-ci.yml | 4 ++-- test/web/common_api/common_api_test.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eeddce69e..6deb0a1de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: elixir:1.7.2 services: - name: postgres:9.6.2 - command: ["postgres", "-c", "fsync=off"] + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] variables: POSTGRES_DB: pleroma_test @@ -36,4 +36,4 @@ lint: unit-testing: stage: test script: - - mix test --trace + - mix test --trace --preload-modules diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index d26b6e49c..870648fb5 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.CommonAPI.Test do +defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Web.CommonAPI alias Pleroma.User From 6a8a8e90dafdd905fbcec3dd0159b7931b8642c2 Mon Sep 17 00:00:00 2001 From: lambda Date: Sun, 17 Feb 2019 17:01:22 +0000 Subject: [PATCH 29/34] Update Differences-in-MastodonAPI-Responses.md --- docs/Differences-in-MastodonAPI-Responses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md index 488dc9389..f6a5b6461 100644 --- a/docs/Differences-in-MastodonAPI-Responses.md +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -8,4 +8,4 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas ## Attachment cap -Some apps operate under the assumption that no more than 4 attachments ccan be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. +Some apps operate under the assumption that no more than 4 attachments can be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting. From 94708d63705f1e4e7f3aaebb6744634237b0cf21 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 17 Feb 2019 23:57:35 +0300 Subject: [PATCH 30/34] Render only "id", "valid_until" and "app_name" in TokenView --- lib/pleroma/web/oauth/token.ex | 1 + lib/pleroma/web/twitter_api/views/token_view.ex | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index f5594f834..7fe58f6a2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -68,5 +68,6 @@ defmodule Pleroma.Web.OAuth.Token do where: t.user_id == ^user_id ) |> Repo.all() + |> Repo.preload(:app) end end diff --git a/lib/pleroma/web/twitter_api/views/token_view.ex b/lib/pleroma/web/twitter_api/views/token_view.ex index 96b8526a4..3ff314913 100644 --- a/lib/pleroma/web/twitter_api/views/token_view.ex +++ b/lib/pleroma/web/twitter_api/views/token_view.ex @@ -14,9 +14,8 @@ defmodule Pleroma.Web.TwitterAPI.TokenView do def render("show.json", %{token: token_entry}) do %{ id: token_entry.id, - token: token_entry.token, - refresh_token: token_entry.refresh_token, - valid_until: token_entry.valid_until + valid_until: token_entry.valid_until, + app_name: token_entry.app.client_name } end end From fd17a0cc9b78d1338e1fee51aa452858172639fe Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 18 Feb 2019 00:10:48 +0300 Subject: [PATCH 31/34] Fix test --- test/web/twitter_api/twitter_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 527a920fb..3922b3c5e 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1896,7 +1896,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> hd() |> Map.keys() - assert keys -- ["id", "refresh_token", "token", "valid_until"] == [] + assert keys -- ["id", "app_name", "valid_until"] == [] end test "revoke token", %{token: token} do From fc35481445c4fdfea9fab58a74c5f4231b56885b Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 19 Feb 2019 10:43:37 +0300 Subject: [PATCH 32/34] Update user cache when user tags are updated --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ff84e7b0a..322c338cd 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1200,7 +1200,7 @@ defmodule Pleroma.User do {:ok, updated_user} = user |> change(%{tags: new_tags}) - |> Repo.update() + |> update_and_set_cache() updated_user end From 96dcacade1fb5efd3c5118a1b8510e0cedbfeb85 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 19 Feb 2019 13:23:13 +0300 Subject: [PATCH 33/34] properly check for follower address in is_private? --- lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ab2872f56..839b6ce2a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -876,7 +876,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def is_private?(activity) do - !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers")) + unless is_public?(activity) do + follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address + Enum.any?(activity.data["to"], &(&1 == follower_address)) + else + false + end end def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true From 109b01a6318e01566a3a148c9c9ecd0313546f5f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 19 Feb 2019 13:52:15 +0300 Subject: [PATCH 34/34] mark ap_id unique_constraint --- lib/pleroma/user.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29d2b3d89..c2ca58fb9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -237,6 +237,7 @@ defmodule Pleroma.User do changeset |> put_change(:password_hash, hashed) |> put_change(:ap_id, ap_id) + |> unique_constraint(:ap_id) |> put_change(:following, [followers]) |> put_change(:follower_address, followers) else