From 0c2b09a9ba771b3b04a0a08ed940823bd8601a9f Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Fri, 8 May 2020 22:08:11 +0300 Subject: [PATCH 01/10] Add migration for counter_cache table update --- .../20200508092434_update_counter_cache_table.exs | 144 +++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 priv/repo/migrations/20200508092434_update_counter_cache_table.exs diff --git a/priv/repo/migrations/20200508092434_update_counter_cache_table.exs b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs new file mode 100644 index 000000000..81a8d6397 --- /dev/null +++ b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs @@ -0,0 +1,144 @@ +defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do + use Ecto.Migration + + @function_name "update_status_visibility_counter_cache" + @trigger_name "status_visibility_counter_cache_trigger" + + def up do + execute("drop trigger if exists #{@trigger_name} on activities") + execute("drop function if exists #{@function_name}()") + drop_if_exists(unique_index(:counter_cache, [:name])) + drop_if_exists(table(:counter_cache)) + + create_if_not_exists table(:counter_cache) do + add(:instance, :string, null: false) + add(:direct, :bigint, null: false, default: 0) + add(:private, :bigint, null: false, default: 0) + add(:unlisted, :bigint, null: false, default: 0) + add(:public, :bigint, null: false, default: 0) + end + + create_if_not_exists(unique_index(:counter_cache, [:instance])) + + """ + CREATE OR REPLACE FUNCTION #{@function_name}() + RETURNS TRIGGER AS + $$ + DECLARE + token_id smallint; + hostname character varying(255); + visibility_new character varying(64); + visibility_old character varying(64); + actor character varying(255); + BEGIN + SELECT "tokid" INTO "token_id" FROM ts_token_type('default') WHERE "alias" = 'host'; + IF TG_OP = 'DELETE' THEN + actor := OLD.actor; + ELSE + actor := NEW.actor; + END IF; + SELECT "token" INTO "hostname" FROM ts_parse('default', actor) WHERE "tokid" = token_id; + IF hostname IS NULL THEN + hostname := split_part(actor, '/', 3); + END IF; + IF TG_OP = 'INSERT' THEN + visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data); + IF NEW.data->>'type' = 'Create' THEN + EXECUTE format('INSERT INTO "counter_cache" ("instance", %1$I) VALUES ($1, 1) + ON CONFLICT ("instance") DO + UPDATE SET %1$I = "counter_cache".%1$I + 1', visibility_new) + USING hostname; + END IF; + RETURN NEW; + ELSIF TG_OP = 'UPDATE' THEN + visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data); + visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data); + IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and visibility_new != visibility_old THEN + EXECUTE format('UPDATE "counter_cache" SET + %1$I = greatest("counter_cache".%1$I - 1, 0), + %2$I = "counter_cache".%2$I + 1 + WHERE "instance" = $1', visibility_old, visibility_new) + USING hostname; + END IF; + RETURN NEW; + ELSIF TG_OP = 'DELETE' THEN + IF OLD.data->>'type' = 'Create' THEN + visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data); + EXECUTE format('UPDATE "counter_cache" SET + %1$I = greatest("counter_cache".%1$I - 1, 0) + WHERE "instance" = $1', visibility_old) + USING hostname; + END IF; + RETURN OLD; + END IF; + END; + $$ + LANGUAGE 'plpgsql'; + """ + |> execute() + + execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities") + + """ + CREATE TRIGGER #{@trigger_name} + BEFORE + INSERT + OR UPDATE of recipients, data + OR DELETE + ON activities + FOR EACH ROW + EXECUTE PROCEDURE #{@function_name}(); + """ + |> execute() + end + + def down do + execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities") + execute("DROP FUNCTION IF EXISTS #{@function_name}()") + drop_if_exists(unique_index(:counter_cache, [:instance])) + drop_if_exists(table(:counter_cache)) + + create_if_not_exists table(:counter_cache) do + add(:name, :string, null: false) + add(:count, :bigint, null: false, default: 0) + end + + create_if_not_exists(unique_index(:counter_cache, [:name])) + + """ + CREATE OR REPLACE FUNCTION #{@function_name}() + RETURNS TRIGGER AS + $$ + DECLARE + BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.data->>'type' = 'Create' THEN + EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1'; + END IF; + RETURN NEW; + ELSIF TG_OP = 'UPDATE' THEN + IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN + EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1'; + EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';'; + END IF; + RETURN NEW; + ELSIF TG_OP = 'DELETE' THEN + IF OLD.data->>'type' = 'Create' THEN + EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';'; + END IF; + RETURN OLD; + END IF; + END; + $$ + LANGUAGE 'plpgsql'; + """ + |> execute() + + """ + CREATE TRIGGER #{@trigger_name} BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities + FOR EACH ROW + EXECUTE PROCEDURE #{@function_name}(); + """ + |> execute() + end +end From 39d2f2118aed7906cb352d8a37f22da73f3a3aa3 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 01:20:50 +0300 Subject: [PATCH 02/10] update counter_cache logic --- lib/mix/tasks/pleroma/refresh_counter_cache.ex | 42 +++++++++++----- lib/pleroma/counter_cache.ex | 66 ++++++++++++++++++++------ lib/pleroma/stats.ex | 8 +--- 3 files changed, 82 insertions(+), 34 deletions(-) diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex index 15b4dbfa6..280201bef 100644 --- a/lib/mix/tasks/pleroma/refresh_counter_cache.ex +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -17,30 +17,46 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do def run([]) do Mix.Pleroma.start_pleroma() - ["public", "unlisted", "private", "direct"] - |> Enum.each(fn visibility -> - count = status_visibility_count_query(visibility) - name = "status_visibility_#{visibility}" - CounterCache.set(name, count) - Mix.Pleroma.shell_info("Set #{name} to #{count}") + Activity + |> distinct([a], true) + |> select([a], fragment("split_part(?, '/', 3)", a.actor)) + |> Repo.all() + |> Enum.each(fn instance -> + counters = instance_counters(instance) + CounterCache.set(instance, counters) + Mix.Pleroma.shell_info("Setting #{instance} counters: #{inspect(counters)}") end) Mix.Pleroma.shell_info("Done") end - defp status_visibility_count_query(visibility) do + defp instance_counters(instance) do + counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0} + Activity - |> where( + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([a], like(a.actor, ^"%#{instance}%")) + |> select( + [a], + {fragment( + "activity_visibility(?, ?, ?)", + a.actor, + a.recipients, + a.data + ), count(a.id)} + ) + |> group_by( [a], fragment( - "activity_visibility(?, ?, ?) = ?", + "activity_visibility(?, ?, ?)", a.actor, a.recipients, - a.data, - ^visibility + a.data ) ) - |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30)) + |> Repo.all(timeout: :timer.minutes(30)) + |> Enum.reduce(counters, fn {visibility, count}, acc -> + Map.put(acc, visibility, count) + end) end end diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex index 4d348a413..b469e7b50 100644 --- a/lib/pleroma/counter_cache.ex +++ b/lib/pleroma/counter_cache.ex @@ -10,32 +10,70 @@ defmodule Pleroma.CounterCache do import Ecto.Query schema "counter_cache" do - field(:name, :string) - field(:count, :integer) + field(:instance, :string) + field(:public, :integer) + field(:unlisted, :integer) + field(:private, :integer) + field(:direct, :integer) end def changeset(struct, params) do struct - |> cast(params, [:name, :count]) - |> validate_required([:name]) - |> unique_constraint(:name) + |> cast(params, [:instance, :public, :unlisted, :private, :direct]) + |> validate_required([:instance]) + |> unique_constraint(:instance) end - def get_as_map(names) when is_list(names) do + def get_by_instance(instance) do CounterCache - |> where([cc], cc.name in ^names) - |> Repo.all() - |> Enum.group_by(& &1.name, & &1.count) - |> Map.new(fn {k, v} -> {k, hd(v)} end) + |> select([c], %{ + "public" => c.public, + "unlisted" => c.unlisted, + "private" => c.private, + "direct" => c.direct + }) + |> where([c], c.instance == ^instance) + |> Repo.one() + |> case do + nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0} + val -> val + end end - def set(name, count) do + def get_as_map() do + CounterCache + |> select([c], %{ + "public" => sum(c.public), + "unlisted" => sum(c.unlisted), + "private" => sum(c.private), + "direct" => sum(c.direct) + }) + |> Repo.one() + end + + def set(instance, values) do + params = + Enum.reduce( + ["public", "private", "unlisted", "direct"], + %{"instance" => instance}, + fn param, acc -> + Map.put_new(acc, param, Map.get(values, param, 0)) + end + ) + %CounterCache{} - |> changeset(%{"name" => name, "count" => count}) + |> changeset(params) |> Repo.insert( - on_conflict: [set: [count: count]], + on_conflict: [ + set: [ + public: params["public"], + private: params["private"], + unlisted: params["unlisted"], + direct: params["direct"] + ] + ], returning: true, - conflict_target: :name + conflict_target: :instance ) end end diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 6b3a8a41f..4e355bd5c 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -98,13 +98,7 @@ defmodule Pleroma.Stats do end def get_status_visibility_count do - counter_cache = - CounterCache.get_as_map([ - "status_visibility_public", - "status_visibility_private", - "status_visibility_unlisted", - "status_visibility_direct" - ]) + counter_cache = CounterCache.get_as_map() %{ public: counter_cache["status_visibility_public"] || 0, From cbe383ae832f13d5d2a20ee8fb5e85498205fbc3 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 11:30:37 +0300 Subject: [PATCH 03/10] Update stats admin endpoint --- lib/pleroma/counter_cache.ex | 6 ++- lib/pleroma/stats.ex | 15 +++--- lib/pleroma/web/admin_api/admin_api_controller.ex | 7 ++- .../20200508092434_update_counter_cache_table.exs | 8 +++- test/stats_test.exs | 55 ++++++++++++++++++---- test/web/admin_api/admin_api_controller_test.exs | 20 ++++++++ 6 files changed, 87 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex index b469e7b50..a940b5e50 100644 --- a/lib/pleroma/counter_cache.ex +++ b/lib/pleroma/counter_cache.ex @@ -40,7 +40,7 @@ defmodule Pleroma.CounterCache do end end - def get_as_map() do + def get_sum() do CounterCache |> select([c], %{ "public" => sum(c.public), @@ -49,6 +49,10 @@ defmodule Pleroma.CounterCache do "direct" => sum(c.direct) }) |> Repo.one() + |> Enum.map(fn {visibility, dec_count} -> + {visibility, Decimal.to_integer(dec_count)} + end) + |> Enum.into(%{}) end def set(instance, values) do diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 4e355bd5c..9a03f01db 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -97,14 +97,11 @@ defmodule Pleroma.Stats do } end - def get_status_visibility_count do - counter_cache = CounterCache.get_as_map() - - %{ - public: counter_cache["status_visibility_public"] || 0, - unlisted: counter_cache["status_visibility_unlisted"] || 0, - private: counter_cache["status_visibility_private"] || 0, - direct: counter_cache["status_visibility_direct"] || 0 - } + def get_status_visibility_count(instance \\ nil) do + if is_nil(instance) do + CounterCache.get_sum() + else + CounterCache.get_by_instance(instance) + end end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 9f1fd3aeb..4db9f4cac 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -1122,11 +1122,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - def stats(conn, _) do - count = Stats.get_status_visibility_count() + def stats(conn, params) do + counters = Stats.get_status_visibility_count(params["instance"]) - conn - |> json(%{"status_visibility" => count}) + json(conn, %{"status_visibility" => counters}) end defp errors(conn, {:error, :not_found}) do diff --git a/priv/repo/migrations/20200508092434_update_counter_cache_table.exs b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs index 81a8d6397..3d9bfc877 100644 --- a/priv/repo/migrations/20200508092434_update_counter_cache_table.exs +++ b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs @@ -43,7 +43,8 @@ defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do END IF; IF TG_OP = 'INSERT' THEN visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data); - IF NEW.data->>'type' = 'Create' THEN + IF NEW.data->>'type' = 'Create' + AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN EXECUTE format('INSERT INTO "counter_cache" ("instance", %1$I) VALUES ($1, 1) ON CONFLICT ("instance") DO UPDATE SET %1$I = "counter_cache".%1$I + 1', visibility_new) @@ -53,7 +54,10 @@ defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do ELSIF TG_OP = 'UPDATE' THEN visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data); visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data); - IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and visibility_new != visibility_old THEN + IF (NEW.data->>'type' = 'Create') + AND (OLD.data->>'type' = 'Create') + AND visibility_new != visibility_old + AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN EXECUTE format('UPDATE "counter_cache" SET %1$I = greatest("counter_cache".%1$I - 1, 0), %2$I = "counter_cache".%2$I + 1 diff --git a/test/stats_test.exs b/test/stats_test.exs index c1aeb2c7f..33ed0b7dd 100644 --- a/test/stats_test.exs +++ b/test/stats_test.exs @@ -17,10 +17,11 @@ defmodule Pleroma.StatsTest do end end - describe "status visibility count" do + describe "status visibility sum count" do test "on new status" do + instance2 = "instance2.tld" user = insert(:user) - other_user = insert(:user) + other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"}) CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) @@ -45,24 +46,24 @@ defmodule Pleroma.StatsTest do }) end) - assert %{direct: 3, private: 4, public: 1, unlisted: 2} = + assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} = Pleroma.Stats.get_status_visibility_count() end test "on status delete" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) - assert %{public: 1} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count() CommonAPI.delete(activity.id, user) - assert %{public: 0} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count() end test "on status visibility update" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) - assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count() {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"}) - assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count() end test "doesn't count unrelated activities" do @@ -73,8 +74,46 @@ defmodule Pleroma.StatsTest do CommonAPI.favorite(other_user, activity.id) CommonAPI.repeat(activity.id, other_user) - assert %{direct: 0, private: 0, public: 1, unlisted: 0} = + assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} = Pleroma.Stats.get_status_visibility_count() end end + + describe "status visibility by instance count" do + test "single instance" do + local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1) + instance2 = "instance2.tld" + user1 = insert(:user) + user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"}) + + CommonAPI.post(user1, %{"visibility" => "public", "status" => "hey"}) + + Enum.each(1..5, fn _ -> + CommonAPI.post(user1, %{ + "visibility" => "unlisted", + "status" => "hey" + }) + end) + + Enum.each(1..10, fn _ -> + CommonAPI.post(user1, %{ + "visibility" => "direct", + "status" => "hey @#{user2.nickname}" + }) + end) + + Enum.each(1..20, fn _ -> + CommonAPI.post(user2, %{ + "visibility" => "private", + "status" => "hey" + }) + end) + + assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} = + Pleroma.Stats.get_status_visibility_count(local_instance) + + assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} = + Pleroma.Stats.get_status_visibility_count(instance2) + end + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4697af50e..c3de89ac0 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -3612,6 +3612,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} = response["status_visibility"] end + + test "by instance", %{conn: conn} do + admin = insert(:user, is_admin: true) + user1 = insert(:user) + instance2 = "instance2.tld" + user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"}) + + CommonAPI.post(user1, %{"visibility" => "public", "status" => "hey"}) + CommonAPI.post(user2, %{"visibility" => "unlisted", "status" => "hey"}) + CommonAPI.post(user2, %{"visibility" => "private", "status" => "hey"}) + + response = + conn + |> assign(:user, admin) + |> get("/api/pleroma/admin/stats", instance: instance2) + |> json_response(200) + + assert %{"direct" => 0, "private" => 1, "public" => 0, "unlisted" => 1} = + response["status_visibility"] + end end describe "POST /api/pleroma/admin/oauth_app" do From 01b06d6dbfdeff7e1733d575fb94eee4dafbb56a Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 11:43:31 +0300 Subject: [PATCH 04/10] Show progress in refresh_counter_cache task --- lib/mix/tasks/pleroma/refresh_counter_cache.ex | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex index 280201bef..b44e2545d 100644 --- a/lib/mix/tasks/pleroma/refresh_counter_cache.ex +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -17,14 +17,21 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do def run([]) do Mix.Pleroma.start_pleroma() - Activity - |> distinct([a], true) - |> select([a], fragment("split_part(?, '/', 3)", a.actor)) - |> Repo.all() - |> Enum.each(fn instance -> + instances = + Activity + |> distinct([a], true) + |> select([a], fragment("split_part(?, '/', 3)", a.actor)) + |> Repo.all() + + instances + |> Enum.with_index(1) + |> Enum.each(fn {instance, i} -> counters = instance_counters(instance) CounterCache.set(instance, counters) - Mix.Pleroma.shell_info("Setting #{instance} counters: #{inspect(counters)}") + + Mix.Pleroma.shell_info( + "[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}" + ) end) Mix.Pleroma.shell_info("Done") From 5c368b004b1a736836d4bc9f68c54714a33056cd Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 11:49:54 +0300 Subject: [PATCH 05/10] Fix refresh_counter_cache test --- test/tasks/refresh_counter_cache_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tasks/refresh_counter_cache_test.exs b/test/tasks/refresh_counter_cache_test.exs index b63f44c08..378664148 100644 --- a/test/tasks/refresh_counter_cache_test.exs +++ b/test/tasks/refresh_counter_cache_test.exs @@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n" - assert %{direct: 3, private: 4, public: 1, unlisted: 2} = + assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} = Pleroma.Stats.get_status_visibility_count() end end From 4f265397179e7286f27fafaf8365a0edc6972448 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 11:59:49 +0300 Subject: [PATCH 06/10] Fix credo warning --- lib/pleroma/counter_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex index a940b5e50..aa6d38687 100644 --- a/lib/pleroma/counter_cache.ex +++ b/lib/pleroma/counter_cache.ex @@ -40,7 +40,7 @@ defmodule Pleroma.CounterCache do end end - def get_sum() do + def get_sum do CounterCache |> select([c], %{ "public" => sum(c.public), From 56819f7f0604e5d4eb69edba1d6828256acbc7fe Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 13:13:26 +0300 Subject: [PATCH 07/10] Use index on refresh_counter_cache --- lib/mix/tasks/pleroma/refresh_counter_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex index b44e2545d..efcbaa3b1 100644 --- a/lib/mix/tasks/pleroma/refresh_counter_cache.ex +++ b/lib/mix/tasks/pleroma/refresh_counter_cache.ex @@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do Activity |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> where([a], like(a.actor, ^"%#{instance}%")) + |> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance)) |> select( [a], {fragment( From 4c197023903a183790fb2dc67b5637bfabd53bcb Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 9 May 2020 14:32:08 +0300 Subject: [PATCH 08/10] Add docs --- CHANGELOG.md | 12 ++++++++++++ docs/API/admin_api.md | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d469793f0..f2c9e106e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed
API Changes + - **Breaking:** Emoji API: changed methods and renamed routes.
+
+ Admin API Changes + +- Status visibility stats: now can return stats per instance. + +- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`) +
+ ### Removed - **Breaking:** removed `with_move` parameter from notifications timeline. @@ -76,6 +85,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 2. Run database migrations (inside Pleroma directory): - OTP: `./bin/pleroma_ctl migrate` - From Source: `mix ecto.migrate` +3. Reset status visibility counters (inside Pleroma directory): + - OTP: `./bin/pleroma_ctl refresh_counter_cache` + - From Source: `mix pleroma.refresh_counter_cache` ## [2.0.2] - 2020-04-08 diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index c455047cc..fa74e7460 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1096,6 +1096,10 @@ Loads json generated from `config/descriptions.exs`. ### Stats +- Query Params: + - *optional* `instance`: **string** instance hostname (without protocol) to get stats for +- Example: `https://mypleroma.org/api/pleroma/admin/stats?instance=lain.com` + - Response: ```json From f3f8ed9e19f1ab8863141ba8b31773c54f4771fb Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 10 May 2020 09:13:24 +0300 Subject: [PATCH 09/10] Set sum types in query --- lib/pleroma/counter_cache.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex index aa6d38687..ebd1f603d 100644 --- a/lib/pleroma/counter_cache.ex +++ b/lib/pleroma/counter_cache.ex @@ -43,16 +43,12 @@ defmodule Pleroma.CounterCache do def get_sum do CounterCache |> select([c], %{ - "public" => sum(c.public), - "unlisted" => sum(c.unlisted), - "private" => sum(c.private), - "direct" => sum(c.direct) + "public" => type(sum(c.public), :integer), + "unlisted" => type(sum(c.unlisted), :integer), + "private" => type(sum(c.private), :integer), + "direct" => type(sum(c.direct), :integer) }) |> Repo.one() - |> Enum.map(fn {visibility, dec_count} -> - {visibility, Decimal.to_integer(dec_count)} - end) - |> Enum.into(%{}) end def set(instance, values) do From 1be6b3056e97654612f377eaf3c8d80de6d8d77f Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Mon, 18 May 2020 12:38:16 +0300 Subject: [PATCH 10/10] Use indexed split_part/3 to get a hostname rather than ts_ functions --- priv/repo/migrations/20200508092434_update_counter_cache_table.exs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/priv/repo/migrations/20200508092434_update_counter_cache_table.exs b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs index 3d9bfc877..738344868 100644 --- a/priv/repo/migrations/20200508092434_update_counter_cache_table.exs +++ b/priv/repo/migrations/20200508092434_update_counter_cache_table.exs @@ -25,22 +25,17 @@ defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do RETURNS TRIGGER AS $$ DECLARE - token_id smallint; hostname character varying(255); visibility_new character varying(64); visibility_old character varying(64); actor character varying(255); BEGIN - SELECT "tokid" INTO "token_id" FROM ts_token_type('default') WHERE "alias" = 'host'; IF TG_OP = 'DELETE' THEN actor := OLD.actor; ELSE actor := NEW.actor; END IF; - SELECT "token" INTO "hostname" FROM ts_parse('default', actor) WHERE "tokid" = token_id; - IF hostname IS NULL THEN - hostname := split_part(actor, '/', 3); - END IF; + hostname := split_part(actor, '/', 3); IF TG_OP = 'INSERT' THEN visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data); IF NEW.data->>'type' = 'Create'