From 9717166d105cec6f9c878653aec2d409265e80a6 Mon Sep 17 00:00:00 2001 From: eal Date: Sat, 13 Jan 2018 18:24:16 +0200 Subject: [PATCH 1/3] Add a stats agent for storing data from expensive queries. --- lib/pleroma/application.ex | 1 + lib/pleroma/stats.ex | 38 ++++++++++++++++++++++ .../web/mastodon_api/mastodon_api_controller.ex | 12 +++++-- lib/pleroma/web/router.ex | 1 + 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 lib/pleroma/stats.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2969ca3c4..cdfca8b1a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -21,6 +21,7 @@ defmodule Pleroma.Application do ]]), worker(Pleroma.Web.Federator, []), worker(Pleroma.Web.ChatChannel.ChatChannelState, []), + worker(Pleroma.Stats, []), ] ++ if Mix.env == :test, do: [], else: [worker(Pleroma.Web.Streamer, [])] diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex new file mode 100644 index 000000000..45fa27b55 --- /dev/null +++ b/lib/pleroma/stats.ex @@ -0,0 +1,38 @@ +defmodule Pleroma.Stats do + use Agent + import Ecto.Query + alias Pleroma.{User, Repo, Activity} + + def start_link do + agent = Agent.start_link(fn -> %{} end, name: __MODULE__) + schedule_update() + agent + end + + def get do + Agent.get(__MODULE__, fn stats -> stats end) + end + + def schedule_update do + update_stats() + spawn(fn -> + Process.sleep(1000 * 60 * 60 * 1) # 1 hour + schedule_update() + end) + end + + def update_stats do + peers = from(u in Pleroma.User, + select: fragment("?->'host'", u.info), + where: u.local != ^true) + |> Repo.all() |> Enum.uniq() + domain_count = Enum.count(peers) + status_query = from p in Activity, + where: p.local == ^true, + where: fragment("?->'object'->>'type' = ?", p.data, ^"Note") + status_count = Repo.aggregate(status_query, :count, :id) + Agent.update(__MODULE__, fn _ -> + %{peers: peers, domain_count: domain_count, status_count: status_count} + end) + end +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index ffa6f42c8..6ca7a6076 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - alias Pleroma.{Repo, Activity, User, Notification} + alias Pleroma.{Repo, Activity, User, Notification, Stats} alias Pleroma.Web alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -94,6 +94,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def masto_instance(conn, _params) do user_count = Repo.aggregate(User.local_user_query, :count, :id) + %{domain_count: domain_count, status_count: status_count} = Stats.get() response = %{ uri: Web.base_url, title: Keyword.get(@instance, :name), @@ -104,9 +105,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do streaming_api: String.replace(Web.base_url, ["http","https"], "wss") }, stats: %{ - status_count: 2, + status_count: status_count, user_count: user_count, - domain_count: 3 + domain_count: domain_count }, max_toot_chars: Keyword.get(@instance, :limit) } @@ -114,6 +115,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do json(conn, response) end + def peers(conn, _params) do + %{peers: peers} = Stats.get() + json(conn, peers) + end + defp mastodonized_emoji do Pleroma.Formatter.get_custom_emoji() |> Enum.map(fn {shortcode, relative_url} -> diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8e6681e68..09104fc86 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -106,6 +106,7 @@ defmodule Pleroma.Web.Router do scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through :api get "/instance", MastodonAPIController, :masto_instance + get "/instance/peers", MastodonAPIController, :peers post "/apps", MastodonAPIController, :create_app get "/custom_emojis", MastodonAPIController, :custom_emojis From d8db39564eb21d9d9b572baae1886af8d0c047d5 Mon Sep 17 00:00:00 2001 From: eal Date: Sun, 14 Jan 2018 08:15:11 +0200 Subject: [PATCH 2/3] Move user count to stats agent. --- lib/pleroma/stats.ex | 13 +++++++++---- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 11 ++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 45fa27b55..938e5c343 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -4,13 +4,17 @@ defmodule Pleroma.Stats do alias Pleroma.{User, Repo, Activity} def start_link do - agent = Agent.start_link(fn -> %{} end, name: __MODULE__) + agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__) schedule_update() agent end - def get do - Agent.get(__MODULE__, fn stats -> stats end) + def get_stats do + Agent.get(__MODULE__, fn {_, stats} -> stats end) + end + + def get_peers do + Agent.get(__MODULE__, fn {peers, _} -> peers end) end def schedule_update do @@ -31,8 +35,9 @@ defmodule Pleroma.Stats do where: p.local == ^true, where: fragment("?->'object'->>'type' = ?", p.data, ^"Note") status_count = Repo.aggregate(status_query, :count, :id) + user_count = Repo.aggregate(User.local_user_query, :count, :id) Agent.update(__MODULE__, fn _ -> - %{peers: peers, domain_count: domain_count, status_count: status_count} + {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}} end) end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 6ca7a6076..0615ac11a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -93,8 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do @instance Application.get_env(:pleroma, :instance) def masto_instance(conn, _params) do - user_count = Repo.aggregate(User.local_user_query, :count, :id) - %{domain_count: domain_count, status_count: status_count} = Stats.get() response = %{ uri: Web.base_url, title: Keyword.get(@instance, :name), @@ -104,11 +102,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do urls: %{ streaming_api: String.replace(Web.base_url, ["http","https"], "wss") }, - stats: %{ - status_count: status_count, - user_count: user_count, - domain_count: domain_count - }, + stats: Stats.get_stats, max_toot_chars: Keyword.get(@instance, :limit) } @@ -116,8 +110,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def peers(conn, _params) do - %{peers: peers} = Stats.get() - json(conn, peers) + json(conn, Stats.get_peers) end defp mastodonized_emoji do From 6e1cb86166d82a4ac4e5fb2ecf8955b18a7ab466 Mon Sep 17 00:00:00 2001 From: eal Date: Sun, 14 Jan 2018 08:28:35 +0200 Subject: [PATCH 3/3] Fix instance test. --- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 79ad671d9..93b29dfae 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -586,11 +586,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do {:ok, _} = TwitterAPI.create_status(user, %{"status" => "cofe"}) + Pleroma.Stats.update_stats() + conn = conn |> get("/api/v1/instance") assert result = json_response(conn, 200) assert result["stats"]["user_count"] == 2 + assert result["stats"]["status_count"] == 1 end end