From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: [PATCH] sync with develop --- benchmarks/load_testing/activities.ex | 42 +++++++++++++++++ benchmarks/load_testing/helper.ex | 11 +++++ benchmarks/load_testing/users.ex | 61 ++++++++++++++----------- benchmarks/mix/tasks/pleroma/benchmarks/tags.ex | 24 ++++------ benchmarks/mix/tasks/pleroma/load_testing.ex | 10 +--- lib/mix/pleroma.ex | 1 + 6 files changed, 99 insertions(+), 50 deletions(-) diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index db0e5a66f..121d5c500 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -85,6 +85,48 @@ defmodule Pleroma.LoadTesting.Activities do :ok end + def generate_power_intervals(opts \\ []) do + count = Keyword.get(opts, :count, 20) + power = Keyword.get(opts, :power, 2) + IO.puts("Generating #{count} intervals for a power #{power} series...") + counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) + sum = Enum.sum(counts) + + densities = + Enum.map(counts, fn c -> + c / sum + end) + + densities + |> Enum.reduce(0, fn density, acc -> + if acc == 0 do + [{0, density}] + else + [{_, lower} | _] = acc + [{lower, lower + density} | acc] + end + end) + |> Enum.reverse() + end + + def generate_tagged_activities(opts \\ []) do + tag_count = Keyword.get(opts, :tag_count, 20) + users = Keyword.get(opts, :users, Repo.all(Pleroma.User)) + activity_count = Keyword.get(opts, :count, 200_000) + + intervals = generate_power_intervals(count: tag_count) + + IO.puts( + "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" + ) + + Enum.each(1..activity_count, fn _ -> + random = :rand.uniform() + i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) + CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + end) + end + defp generate_long_thread(visibility, user, friends, non_friends, _opts) do group = if visibility == "public", diff --git a/benchmarks/load_testing/helper.ex b/benchmarks/load_testing/helper.ex index 23bbb1cec..cab60acb4 100644 --- a/benchmarks/load_testing/helper.ex +++ b/benchmarks/load_testing/helper.ex @@ -1,3 +1,14 @@ defmodule Pleroma.LoadTesting.Helper do + alias Ecto.Adapters.SQL + alias Pleroma.Repo + def to_sec(microseconds), do: microseconds / 1_000_000 + + def clean_tables do + IO.puts("Deleting old data...\n") + SQL.query!(Repo, "TRUNCATE users CASCADE;") + SQL.query!(Repo, "TRUNCATE activities CASCADE;") + SQL.query!(Repo, "TRUNCATE objects CASCADE;") + SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;") + end end diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index 951b30d91..bc31dc08b 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -20,31 +20,31 @@ defmodule Pleroma.LoadTesting.Users do def generate(opts \\ []) do opts = Keyword.merge(@defaults, opts) - IO.puts("Starting generating #{opts[:users]} users...") - - {time, _} = :timer.tc(fn -> generate_users(opts[:users]) end) - - IO.puts("Generating users take #{to_sec(time)} sec.\n") + generate_users(opts[:users]) main_user = Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) - IO.puts("Starting making friends for #{opts[:friends]} users...") - {time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end) - - IO.puts("Making friends take #{to_sec(time)} sec.\n") + make_friends(main_user, opts[:friends]) Repo.get(User, main_user.id) end - defp generate_users(max) do - Task.async_stream( - 1..max, - &generate_user(&1), - max_concurrency: @max_concurrency, - timeout: 30_000 - ) - |> Stream.run() + def generate_users(max) do + IO.puts("Starting generating #{opts[:users]} users...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..max, + &generate_user(&1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Generating users take #{to_sec(time)} sec.\n") end defp generate_user(i) do @@ -86,18 +86,25 @@ defmodule Pleroma.LoadTesting.Users do Map.merge(user, urls) end - defp make_friends(main_user, max) when is_integer(max) do - number_of_users = - (max / 2) - |> Kernel.trunc() + def make_friends(main_user, max) when is_integer(max) do + IO.puts("Starting making friends for #{opts[:friends]} users...") - main_user - |> get_users(%{limit: number_of_users, local: :local}) - |> run_stream(main_user) + {time, _} = + :timer.tc(fn -> + number_of_users = + (max / 2) + |> Kernel.trunc() - main_user - |> get_users(%{limit: number_of_users, local: :external}) - |> run_stream(main_user) + main_user + |> get_users(%{limit: number_of_users, local: :local}) + |> run_stream(main_user) + + main_user + |> get_users(%{limit: number_of_users, local: :external}) + |> run_stream(main_user) + end) + + IO.puts("Making friends take #{to_sec(time)} sec.\n") end defp make_friends(%User{} = main_user, %User{} = user) do diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex index fd1506907..657403202 100644 --- a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex +++ b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex @@ -1,9 +1,12 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do use Mix.Task - alias Pleroma.Repo - alias Pleroma.LoadTesting.Generator + + import Pleroma.LoadTesting.Helper, only: [clean_tables: 0] import Ecto.Query + alias Pleroma.Repo + alias Pleroma.Web.MastodonAPI.TimelineController + def run(_args) do Mix.Pleroma.start_pleroma() activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) @@ -11,8 +14,8 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do if activities_count == 0 do IO.puts("Did not find any activities, cleaning and generating") clean_tables() - Generator.generate_users(users_max: 10) - Generator.generate_tagged_activities() + Pleroma.LoadTesting.Users.generate_users(10) + Pleroma.LoadTesting.Activities.generate_tagged_activities() else IO.puts("Found #{activities_count} activities, won't generate new ones") end @@ -34,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do Benchee.run( %{ "Hashtag fetching, any" => fn tags -> - Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + TimelineController.hashtag_fetching( %{ "any" => tags }, @@ -44,7 +47,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do end, # Will always return zero results because no overlapping hashtags are generated. "Hashtag fetching, all" => fn tags -> - Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + TimelineController.hashtag_fetching( %{ "all" => tags }, @@ -64,7 +67,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do Benchee.run( %{ "Hashtag fetching" => fn tag -> - Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( + TimelineController.hashtag_fetching( %{ "tag" => tag }, @@ -77,11 +80,4 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do time: 5 ) end - - defp clean_tables do - IO.puts("Deleting old data...\n") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") - Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") - end end diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex index 262300990..72b225f09 100644 --- a/benchmarks/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -1,8 +1,8 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do use Mix.Task import Ecto.Query + import Pleroma.LoadTesting.Helper, only: [clean_tables: 0] - alias Ecto.Adapters.SQL alias Pleroma.Repo alias Pleroma.User @@ -63,12 +63,4 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do Pleroma.LoadTesting.Fetcher.run_benchmarks(user) end - - defp clean_tables do - IO.puts("Deleting old data...\n") - SQL.query!(Repo, "TRUNCATE users CASCADE;") - SQL.query!(Repo, "TRUNCATE activities CASCADE;") - SQL.query!(Repo, "TRUNCATE objects CASCADE;") - SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;") - end end diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..4dfcc32e7 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -5,6 +5,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do + Mix.Task.run("app.start") Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do