From 7ad5c51f23102e89c491a2ef731e108873a09d71 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 9 Jan 2020 22:18:55 +0300 Subject: [PATCH] Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope --- CHANGELOG.md | 2 + docs/API/admin_api.md | 17 ++++++ lib/mix/tasks/pleroma/refresh_counter_cache.ex | 46 ++++++++++++++ lib/pleroma/counter_cache.ex | 41 +++++++++++++ lib/pleroma/stats.ex | 18 ++++++ lib/pleroma/web/admin_api/admin_api_controller.ex | 10 +++- lib/pleroma/web/router.ex | 1 + .../20200109123126_add_counter_cache_table.exs | 55 +++++++++++++++++ test/stat_test.exs | 70 ++++++++++++++++++++++ test/tasks/refresh_counter_cache_test.exs | 43 +++++++++++++ test/web/admin_api/admin_api_controller_test.exs | 19 ++++++ 11 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/pleroma/refresh_counter_cache.ex create mode 100644 lib/pleroma/counter_cache.ex create mode 100644 priv/repo/migrations/20200109123126_add_counter_cache_table.exs create mode 100644 test/stat_test.exs create mode 100644 test/tasks/refresh_counter_cache_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab098591..08bb7e1c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches. - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation). +- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
API Changes @@ -119,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `reacted` property to `emoji_reactions` - Pleroma API: Add reactions for a single emoji. - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation. +- Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope
### Fixed diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 47acd240e..3882763cd 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -939,3 +939,20 @@ Loads json generated from `config/descriptions.exs`. - Params: - `nicknames` - Response: Array of user nicknames + +## `GET /api/pleroma/admin/stats` + +### Stats + +- Response: + +```json +{ + "status_visibility": { + "direct": 739, + "private": 9, + "public": 17, + "unlisted": 14 + } +} +``` diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex new file mode 100644 index 000000000..bc2571efd --- /dev/null +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.RefreshCounterCache do + @shortdoc "Refreshes counter cache" + + use Mix.Task + + alias Pleroma.Activity + alias Pleroma.CounterCache + alias Pleroma.Repo + + require Logger + import Ecto.Query + + def run([]) do + Mix.Pleroma.start_pleroma() + + ["public", "unlisted", "private", "direct"] + |> Enum.each(fn visibility -> + count = status_visibility_count_query(visibility) + name = "status_visibility_#{visibility}" + CounterCache.set(name, count) + Mix.Pleroma.shell_info("Set #{name} to #{count}") + end) + + Mix.Pleroma.shell_info("Done") + end + + defp status_visibility_count_query(visibility) do + Activity + |> where( + [a], + fragment( + "activity_visibility(?, ?, ?) = ?", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30)) + end +end diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex new file mode 100644 index 000000000..8e868e956 --- /dev/null +++ b/lib/pleroma/counter_cache.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.CounterCache do + alias Pleroma.CounterCache + alias Pleroma.Repo + use Ecto.Schema + import Ecto.Changeset + import Ecto.Query + + schema "counter_cache" do + field(:name, :string) + field(:count, :integer) + end + + def changeset(struct, params) do + struct + |> cast(params, [:name, :count]) + |> validate_required([:name]) + |> unique_constraint(:name) + end + + def get_as_map(names) when is_list(names) do + CounterCache + |> where([cc], cc.name in ^names) + |> Repo.all() + |> Enum.group_by(& &1.name, & &1.count) + |> Map.new(fn {k, v} -> {k, hd(v)} end) + end + + def set(name, count) do + %CounterCache{} + |> changeset(%{"name" => name, "count" => count}) + |> Repo.insert( + on_conflict: [set: [count: count]], + returning: true, + conflict_target: :name + ) + end +end diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index cf590fb01..771a06e32 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Stats do import Ecto.Query + alias Pleroma.CounterCache alias Pleroma.Repo alias Pleroma.User @@ -96,4 +97,21 @@ defmodule Pleroma.Stats do } } end + + def get_status_visibility_count do + counter_cache = + CounterCache.get_as_map([ + "status_visibility_public", + "status_visibility_private", + "status_visibility_unlisted", + "status_visibility_direct" + ]) + + %{ + public: counter_cache["status_visibility_public"] || 0, + unlisted: counter_cache["status_visibility_unlisted"] || 0, + private: counter_cache["status_visibility_private"] || 0, + direct: counter_cache["status_visibility_direct"] || 0 + } + end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 67222ebae..816b8938c 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.ModerationLog alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.ReportNote + alias Pleroma.Stats alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.ActivityPub.ActivityPub @@ -98,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read"], admin: true} - when action in [:config_show, :list_log] + when action in [:config_show, :list_log, :stats] ) plug( @@ -953,6 +954,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do conn |> json("") end + def stats(conn, _) do + count = Stats.get_status_visibility_count() + + conn + |> json(%{"status_visibility" => count}) + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9bfe86704..c2ffb025a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -201,6 +201,7 @@ defmodule Pleroma.Web.Router do get("/moderation_log", AdminAPIController, :list_log) post("/reload_emoji", AdminAPIController, :reload_emoji) + get("/stats", AdminAPIController, :stats) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do diff --git a/priv/repo/migrations/20200109123126_add_counter_cache_table.exs b/priv/repo/migrations/20200109123126_add_counter_cache_table.exs new file mode 100644 index 000000000..df9e21193 --- /dev/null +++ b/priv/repo/migrations/20200109123126_add_counter_cache_table.exs @@ -0,0 +1,55 @@ +defmodule Pleroma.Repo.Migrations.AddCounterCacheTable do + use Ecto.Migration + + def up do + create_if_not_exists table(:counter_cache) do + add(:name, :string, null: false) + add(:count, :bigint, null: false, default: 0) + end + + create_if_not_exists(unique_index(:counter_cache, [:name])) + + """ + CREATE OR REPLACE FUNCTION update_status_visibility_counter_cache() + RETURNS TRIGGER AS + $$ + DECLARE + BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.data->>'type' = 'Create' THEN + EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1'; + END IF; + RETURN NEW; + ELSIF TG_OP = 'UPDATE' THEN + IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN + EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1'; + EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';'; + END IF; + RETURN NEW; + ELSIF TG_OP = 'DELETE' THEN + IF OLD.data->>'type' = 'Create' THEN + EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';'; + END IF; + RETURN OLD; + END IF; + END; + $$ + LANGUAGE 'plpgsql'; + """ + |> execute() + + """ + CREATE TRIGGER status_visibility_counter_cache_trigger BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities + FOR EACH ROW + EXECUTE PROCEDURE update_status_visibility_counter_cache(); + """ + |> execute() + end + + def down do + execute("drop trigger if exists status_visibility_counter_cache_trigger on activities") + execute("drop function if exists update_status_visibility_counter_cache()") + drop_if_exists(unique_index(:counter_cache, [:name])) + drop_if_exists(table(:counter_cache)) + end +end diff --git a/test/stat_test.exs b/test/stat_test.exs new file mode 100644 index 000000000..1f0c6199a --- /dev/null +++ b/test/stat_test.exs @@ -0,0 +1,70 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.StateTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Web.CommonAPI + + describe "status visibility count" do + test "on new status" do + user = insert(:user) + other_user = insert(:user) + + CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + + Enum.each(0..1, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "unlisted", + "status" => "hey" + }) + end) + + Enum.each(0..2, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "direct", + "status" => "hey @#{other_user.nickname}" + }) + end) + + Enum.each(0..3, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "private", + "status" => "hey" + }) + end) + + assert %{direct: 3, private: 4, public: 1, unlisted: 2} = + Pleroma.Stats.get_status_visibility_count() + end + + test "on status delete" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + assert %{public: 1} = Pleroma.Stats.get_status_visibility_count() + CommonAPI.delete(activity.id, user) + assert %{public: 0} = Pleroma.Stats.get_status_visibility_count() + end + + test "on status visibility update" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count() + {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"}) + assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count() + end + + test "doesn't count unrelated activities" do + user = insert(:user) + other_user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + _ = CommonAPI.follow(user, other_user) + CommonAPI.favorite(activity.id, other_user) + CommonAPI.repeat(activity.id, other_user) + + assert %{direct: 0, private: 0, public: 1, unlisted: 0} = + Pleroma.Stats.get_status_visibility_count() + end + end +end diff --git a/test/tasks/refresh_counter_cache_test.exs b/test/tasks/refresh_counter_cache_test.exs new file mode 100644 index 000000000..47367af94 --- /dev/null +++ b/test/tasks/refresh_counter_cache_test.exs @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do + use Pleroma.DataCase + alias Pleroma.Web.CommonAPI + import ExUnit.CaptureIO, only: [capture_io: 1] + import Pleroma.Factory + + test "counts statuses" do + user = insert(:user) + other_user = insert(:user) + + CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + + Enum.each(0..1, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "unlisted", + "status" => "hey" + }) + end) + + Enum.each(0..2, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "direct", + "status" => "hey @#{other_user.nickname}" + }) + end) + + Enum.each(0..3, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "private", + "status" => "hey" + }) + end) + + assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n" + + assert %{direct: 3, private: 4, public: 1, unlisted: 2} = + Pleroma.Stats.get_status_visibility_count() + end +end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 908ef4d37..0b79e4c5c 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -3545,6 +3545,25 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert String.starts_with?(child["group"], ":") assert child["description"] end + + describe "/api/pleroma/admin/stats" do + test "status visibility count", %{conn: conn} do + admin = insert(:user, is_admin: true) + user = insert(:user) + CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"}) + CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"}) + + response = + conn + |> assign(:user, admin) + |> get("/api/pleroma/admin/stats") + |> json_response(200) + + assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} = + response["status_visibility"] + end + end end # Needed for testing