From db690bede92e9ba65b1afaa8605aa5ddb3c992a4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 2 Aug 2019 21:33:12 +0300 Subject: [PATCH 01/13] temp commit --- config/benchmark.exs | 88 ++++++++++++++++++++++++++ lib/load_testing/fetcher.ex | 116 ++++++++++++++++++++++++++++++++++ lib/load_testing/generator.ex | 83 ++++++++++++++++++++++++ lib/load_testing/helper.ex | 16 +++++ lib/mix/tasks/pleroma/load_testing.ex | 100 +++++++++++++++++++++++++++++ 5 files changed, 403 insertions(+) create mode 100644 config/benchmark.exs create mode 100644 lib/load_testing/fetcher.ex create mode 100644 lib/load_testing/generator.ex create mode 100644 lib/load_testing/helper.ex create mode 100644 lib/mix/tasks/pleroma/load_testing.ex diff --git a/config/benchmark.exs b/config/benchmark.exs new file mode 100644 index 000000000..b4f5dbdbd --- /dev/null +++ b/config/benchmark.exs @@ -0,0 +1,88 @@ +use Mix.Config + +# We don't run a server during test. If one is required, +# you can enable the server option below. +config :pleroma, Pleroma.Web.Endpoint, + http: [port: 4001], + url: [port: 4001], + server: true + +# Disable captha for tests +config :pleroma, Pleroma.Captcha, + # It should not be enabled for automatic tests + enabled: false, + # A fake captcha service for tests + method: Pleroma.Captcha.Mock + +# Print only warnings and errors during test +config :logger, level: :warn + +config :pleroma, :auth, oauth_consumer_strategies: [] + +config :pleroma, Pleroma.Upload, filters: [], link_name: false + +config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads" + +config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true + +config :pleroma, :instance, + email: "admin@example.com", + notify_email: "noreply@example.com", + skip_thread_containment: false, + federating: false + +config :pleroma, :activitypub, sign_object_fetches: false + +# Configure your database +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: System.get_env("DB_USER") || "postgres", + database: System.get_env("DB_DATABASE") || "pleroma_test", + hostname: System.get_env("DB_HOST") || "localhost", + # username: "pleroma", + password: + System.get_env("DB_PASS") || + "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", + # password: "", + pool_size: 10, + timeout: 180_000 + +# Reduce hash rounds for testing +config :pbkdf2_elixir, rounds: 1 + +config :tesla, adapter: Tesla.Mock + +config :pleroma, :rich_media, + enabled: false, + ignore_hosts: [], + ignore_tld: ["local", "localdomain", "lan"] + +config :web_push_encryption, :vapid_details, + subject: "mailto:administrator@example.com", + public_key: + "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", + private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" + +config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock + +config :pleroma_job_queue, disabled: true + +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 2, + total_user_limit: 3, + enabled: false + +config :pleroma, :rate_limit, + search: [{1000, 30}, {1000, 30}], + app_account_creation: {10_000, 5}, + password_reset: {1000, 30} + +config :pleroma, :http_security, report_uri: "https://endpoint.com" + +config :pleroma, :http, send_user_agent: false + +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + +config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex new file mode 100644 index 000000000..70c0fcd0c --- /dev/null +++ b/lib/load_testing/fetcher.ex @@ -0,0 +1,116 @@ +defmodule Pleroma.LoadTesting.Fetcher do + use Pleroma.LoadTesting.Helper + + def fetch_user(user) do + IO.puts("=================================") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) + + IO.puts("Query user by id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, ap_id: user.ap_id) + end) + + IO.puts("Query user by ap_id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, email: user.email) + end) + + IO.puts("Query user by email: #{to_sec(time)} sec.") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) + + IO.puts("Query user by nickname: #{to_sec(time)} sec.") + end + + def query_timelines(user) do + IO.puts("\n=================================") + + params = %{ + "count" => 20, + "with_muted" => true, + "type" => ["Create", "Announce"], + "blocking_user" => user, + "muting_user" => user, + "user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) + end) + + IO.puts("Query user home timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "local_only" => true, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + end + + def query_notifications(user) do + IO.puts("\n=================================") + params = %{"count" => "20", "with_muted" => "false"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + + params = %{"count" => "20", "with_muted" => "true"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + end + + def query_long_thread(user, activity) do + IO.puts("\n=================================") + + {time, replies} = + :timer.tc(fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user + } + ) + end) + + IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end +end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex new file mode 100644 index 000000000..a9016b9e8 --- /dev/null +++ b/lib/load_testing/generator.ex @@ -0,0 +1,83 @@ +defmodule Pleroma.LoadTesting.Generator do + use Pleroma.LoadTesting.Helper + + def generate_users(opts) do + IO.puts("Starting generating #{opts[:users_max]} users...") + {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") + end + + defp do_generate_users(opts) do + min = Keyword.get(opts, :users_min, 1) + max = Keyword.get(opts, :users_max) + + query = + "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", + \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" + + users = + Task.async_stream( + min..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) + + query = query <> String.replace_trailing(users, ", \n", ";") + + Ecto.Adapters.SQL.query!(Repo, query) + end + + defp generate_user_data(i) do + user = %User{ + name: "Test テスト User #{i}", + email: "user#{i}@example.com", + nickname: "nick#{i}", + password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + bio: "Tester Number #{i}", + info: %{} + } + + user = %{ + user + | ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + + "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ + user.following + }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ + user.name + }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + end + + def generate_activities(users, opts) do + IO.puts("Starting generating #{opts[:activities_max]} activities...") + {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) + IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_activities(users, opts) do + Task.async_stream( + 1..opts[:activities_max], + fn _ -> + do_generate_activity(users, opts) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_activity(users, opts) do + status = + if opts[:mention], + do: "some status with @#{opts[:mention].nickname}", + else: "some status" + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + end +end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex new file mode 100644 index 000000000..338dba323 --- /dev/null +++ b/lib/load_testing/helper.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.LoadTesting.Helper do + defmacro __using__(_) do + quote do + import Ecto.Query + alias Pleroma.Activity + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub + alias Pleroma.Web.CommonAPI + + defp to_sec(microseconds), do: microseconds / 1_000_000 + end + end +end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex new file mode 100644 index 000000000..9ed30db7e --- /dev/null +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -0,0 +1,100 @@ +defmodule Mix.Tasks.Pleroma.LoadTesting do + use Mix.Task + use Pleroma.LoadTesting.Helper + import Mix.Pleroma + import Pleroma.LoadTesting.Generator + import Pleroma.LoadTesting.Fetcher + + # tODO: remove autovacuum worker until generation is not ended + @shortdoc "Factory for generation data" + @moduledoc """ + Generates data like: + - users + - activities with notifications + + ## Generate data + MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 + MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + + Options: + - `--users NUMBER` - number of users to generate (default: 10000) + - `--activities NUMBER` - number of activities to generate (default: 20000) + """ + + @aliases [u: :users, a: :activities, d: :delete] + @switches [users: :integer, activities: :integer, delete: :boolean] + @users_default 20_000 + @activities_default 50_000 + + def run(args) do + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + start_pleroma() + + current_max = Keyword.get(opts, :users, @users_default) + activities_max = Keyword.get(opts, :activities, @activities_default) + + {users_min, users_max} = + if opts[:delete] do + clean_tables() + {1, current_max} + else + current_count = Repo.aggregate(from(u in User), :count, :id) + 1 + {current_count, current_max + current_count} + end + + opts = + Keyword.put(opts, :users_min, users_min) + |> Keyword.put(:users_max, users_max) + |> Keyword.put(:activities_max, activities_max) + + generate_users(opts) + + # main user for queries + IO.puts("Fetching main user...") + + {time, user} = + :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + + IO.puts("Fetching main user take #{to_sec(time)} sec.\n") + + IO.puts("Fetching users...") + + {time, users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching users take #{to_sec(time)} sec.\n") + + generate_activities(users, opts) + + generate_activities(users, Keyword.put(opts, :mention, user)) + + # generate_replies(user, users, activities) + + # activity = Enum.random(activities) + # generate_long_thread(user, users, activity) + + IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") + IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + query_timelines(user) + query_notifications(user) + # query_long_thread(user, activity) + end + + defp clean_tables do + IO.puts("\n\nDeleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end From 79dde5804427ab24c5bc9aa1b5a44d2259cecc6d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 4 Sep 2019 20:18:11 +0300 Subject: [PATCH 02/13] one more temp commit --- lib/load_testing/fetcher.ex | 120 ++++++++++++------------- lib/load_testing/generator.ex | 127 ++++++++++++++++++++++----- lib/mix/tasks/pleroma/load_testing.ex | 46 ++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++ 4 files changed, 202 insertions(+), 96 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 70c0fcd0c..ed744db9b 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -4,33 +4,18 @@ defmodule Pleroma.LoadTesting.Fetcher do def fetch_user(user) do IO.puts("=================================") - {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) - - IO.puts("Query user by id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, ap_id: user.ap_id) - end) - - IO.puts("Query user by ap_id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, email: user.email) - end) - - IO.puts("Query user by email: #{to_sec(time)} sec.") - - {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) - - IO.puts("Query user by nickname: #{to_sec(time)} sec.") + Benchee.run(%{ + "By id" => fn -> Repo.get_by(User, id: user.id) end, + "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, + "By email" => fn -> Repo.get_by(User, email: user.email) end, + "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end + }) end def query_timelines(user) do IO.puts("\n=================================") - params = %{ + home_timeline_params = %{ "count" => 20, "with_muted" => true, "type" => ["Create", "Announce"], @@ -39,14 +24,7 @@ defmodule Pleroma.LoadTesting.Fetcher do "user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) - end) - - IO.puts("Query user home timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_public_timeline_params = %{ "count" => 20, "local_only" => true, "only_media" => "false", @@ -56,14 +34,7 @@ defmodule Pleroma.LoadTesting.Fetcher do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_federated_timeline_params = %{ "count" => 20, "only_media" => "false", "type" => ["Create", "Announce"], @@ -72,45 +43,76 @@ defmodule Pleroma.LoadTesting.Fetcher do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + Benchee.run(%{ + "User home timeline" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + end, + "User mastodon public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + end, + "User mastodon federated public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + end + }) end def query_notifications(user) do IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "false"} + without_muted_params = %{"count" => "20", "with_muted" => "false"} + with_muted_params = %{"count" => "20", "with_muted" => "true"} - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + Benchee.run(%{ + "Notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + end, + "Notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + end + }) + end - IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + def query_dms(user) do + IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "true"} + params = %{ + "count" => "20", + "with_muted" => "true", + "type" => "Create", + "blocking_user" => user, + "user" => user, + visibility: "direct" + } - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) - - IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + Benchee.run(%{ + "Direct messages with muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + end, + "Direct messages without muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") - {time, replies} = - :timer.tc(fn -> + Benchee.run(%{ + "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], %{ "blocking_user" => user, - "user" => user + "user" => user, + "exclude_id" => activity.id } ) - end) - - IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index a9016b9e8..7f50ee68e 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -8,25 +8,15 @@ defmodule Pleroma.LoadTesting.Generator do end defp do_generate_users(opts) do - min = Keyword.get(opts, :users_min, 1) max = Keyword.get(opts, :users_max) - query = - "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", - \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" - - users = - Task.async_stream( - min..max, - &generate_user_data(&1), - max_concurrency: 10, - timeout: 30_000 - ) - |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) - - query = query <> String.replace_trailing(users, ", \n", ";") - - Ecto.Adapters.SQL.query!(Repo, query) + Task.async_stream( + 1..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.to_list() end defp generate_user_data(i) do @@ -47,11 +37,7 @@ defmodule Pleroma.LoadTesting.Generator do following: [User.ap_id(user)] } - "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ - user.following - }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ - user.name - }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + Pleroma.Repo.insert!(user) end def generate_activities(users, opts) do @@ -80,4 +66,101 @@ defmodule Pleroma.LoadTesting.Generator do Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) end + + def generate_dms(user, users, opts) do + IO.puts("Starting generating #{opts[:dms_max]} DMs") + {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) + IO.puts("Inserting dms take #{to_sec(time)} sec.\n") + end + + defp do_generate_dms(user, users, opts) do + Task.async_stream( + 1..opts[:dms_max], + fn _ -> + do_generate_dm(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_dm(user, users) do + post = %{ + "status" => "@#{user.nickname} some direct message", + "visibility" => "direct" + } + + Pleroma.Web.CommonAPI.post(Enum.random(users), post) + end + + def generate_long_thread(user, users, opts) do + IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + {:ok, activity} + end + + defp do_generate_long_thread(user, users, opts) do + {:ok, %{id: id} = activity} = + Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + + Task.async_stream( + 1..opts[:long_thread_length], + fn _ -> do_generate_thread(users, id) end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + + activity + end + + defp do_generate_thread(users, activity_id) do + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "status" => "reply to main post", + "in_reply_to_status_id" => activity_id + }) + end + + def generate_private_thread(users, opts) do + IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") + {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + end + + defp do_generate_non_visible_posts(users, opts) do + [user1, user2] = Enum.take(users, 2) + {:ok, user1} = Pleroma.User.follow(user1, user2) + {:ok, user2} = Pleroma.User.follow(user2, user1) + + {:ok, activity} = + Pleroma.Web.CommonAPI.post(user1, %{ + "status" => "Some private post", + "visibility" => "private" + }) + + {:ok, activity_public} = + Pleroma.Web.CommonAPI.post(user2, %{ + "status" => "Some public reply", + "in_reply_to_status_id" => activity.id + }) + + Task.async_stream( + 1..opts[:non_visible_posts_max], + fn _ -> do_generate_non_visible_post(users, activity_public) end, + max_concurrency: 10, + timeout: 30_000 + ) + end + + defp do_generate_non_visible_post(users, activity) do + visibility = Enum.random(["private", "public"]) + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "visibility" => visibility, + "status" => "Some #{visibility} reply", + "in_reply_to_status_id" => activity.id + }) + end end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 9ed30db7e..d17cf0b3e 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -21,31 +21,38 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities, d: :delete] - @switches [users: :integer, activities: :integer, delete: :boolean] + @aliases [u: :users, a: :activities] + @switches [ + users: :integer, + activities: :integer, + dms: :integer, + thread_length: :integer, + non_visible_posts: :integer + ] @users_default 20_000 @activities_default 50_000 + @dms_default 50_000 + @thread_length_default 2_000 + @non_visible_posts_default 2_000 def run(args) do - {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) start_pleroma() + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) - current_max = Keyword.get(opts, :users, @users_default) + users_max = Keyword.get(opts, :users, @users_default) activities_max = Keyword.get(opts, :activities, @activities_default) + dms_max = Keyword.get(opts, :dms, @dms_default) + long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) + non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) - {users_min, users_max} = - if opts[:delete] do - clean_tables() - {1, current_max} - else - current_count = Repo.aggregate(from(u in User), :count, :id) + 1 - {current_count, current_max + current_count} - end + clean_tables() opts = - Keyword.put(opts, :users_min, users_min) - |> Keyword.put(:users_max, users_max) + Keyword.put(opts, :users_max, users_max) |> Keyword.put(:activities_max, activities_max) + |> Keyword.put(:dms_max, dms_max) + |> Keyword.put(:long_thread_length, long_thread_length) + |> Keyword.put(:non_visible_posts_max, non_visible_posts) generate_users(opts) @@ -76,6 +83,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do generate_activities(users, Keyword.put(opts, :mention, user)) + generate_dms(user, users, opts) + + {:ok, activity} = generate_long_thread(user, users, opts) + + generate_private_thread(users, opts) + # generate_replies(user, users, activities) # activity = Enum.random(activities) @@ -86,9 +99,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + fetch_user(user) query_timelines(user) query_notifications(user) - # query_long_thread(user, activity) + query_dms(user) + query_long_thread(user, activity) + query_timelines(user) end defp clean_tables do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eeb826814..f2b322314 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -249,6 +249,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) published = params[:published] + quick_insert? = Pleroma.Config.get([:env]) == :benchmark with create_data <- make_create_data( @@ -259,12 +260,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), _ <- increase_poll_votes_if_vote(create_data), + {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), :ok <- maybe_federate(activity) do {:ok, activity} else + {:quick_insert, true, activity} -> + {:ok, activity} + {:fake, true, activity} -> {:ok, activity} From a1125bd5644a5819d1cfbbc8f6b3bb2dadbbc38a Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 5 Sep 2019 16:01:52 +0300 Subject: [PATCH 03/13] generatoin and fetching --- lib/load_testing/fetcher.ex | 126 ++++++++++++++++++++- lib/load_testing/generator.ex | 203 ++++++++++++++++++++++++---------- lib/load_testing/helper.ex | 5 - lib/mix/tasks/pleroma/load_testing.ex | 49 ++++---- 4 files changed, 290 insertions(+), 93 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index ed744db9b..825f921e6 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -51,10 +51,52 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end, "User mastodon public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_public_timeline_params + ) end, "User mastodon federated public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + end + }) + + home_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + + public_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + + public_federated_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + + Benchee.run(%{ + "Rendering home timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: home_activities, + for: user, + as: :activity + }) + end, + "Rendering public timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_activities, + for: user, + as: :activity + }) + end, + "Rendering public federated timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_federated_activities, + for: user, + as: :activity + }) end }) end @@ -72,6 +114,27 @@ defmodule Pleroma.LoadTesting.Fetcher do Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) end }) + + without_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + + with_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + + Benchee.run(%{ + "Render notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: without_muted_notifications, + for: user + }) + end, + "Render notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: with_muted_notifications, + for: user + }) + end + }) end def query_dms(user) do @@ -96,13 +159,40 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) end }) + + dms_with_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + + dms_without_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + + Benchee.run(%{ + "Rendering dms with muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_with_muted, + for: user, + as: :activity + }) + end, + "Rendering dms without muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_without_muted, + for: user, + as: :activity + }) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") Benchee.run(%{ - "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch main post" => fn -> + Pleroma.Activity.get_by_id_with_object(activity.id) + end, "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], @@ -114,5 +204,35 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end }) + + activity = Pleroma.Activity.get_by_id_with_object(activity.id) + + context = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user, + "exclude_id" => activity.id + } + ) + + Benchee.run(%{ + "Render status" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + activity: activity, + for: user + }) + end, + "Render context ancestors" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render( + "index.json", + for: user, + activities: context, + as: :activity + ) + |> Enum.reverse() + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7f50ee68e..6df518aba 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -1,9 +1,11 @@ defmodule Pleroma.LoadTesting.Generator do use Pleroma.LoadTesting.Helper + alias Pleroma.Web.CommonAPI def generate_users(opts) do IO.puts("Starting generating #{opts[:users_max]} users...") {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") end @@ -37,34 +39,111 @@ defmodule Pleroma.LoadTesting.Generator do following: [User.ap_id(user)] } - Pleroma.Repo.insert!(user) + Repo.insert!(user) end - def generate_activities(users, opts) do - IO.puts("Starting generating #{opts[:activities_max]} activities...") - {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) - IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + def generate_activities(user, users) do + do_generate_activities(user, users) end - defp do_generate_activities(users, opts) do - Task.async_stream( - 1..opts[:activities_max], - fn _ -> - do_generate_activity(users, opts) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() + defp do_generate_activities(user, users) do + IO.puts("Starting generating 20000 common activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting common activities take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 20000 activities with mentions...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity_with_mention(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 10000 activities with threads...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn _ -> + do_generate_threads([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n") end - defp do_generate_activity(users, opts) do - status = - if opts[:mention], - do: "some status with @#{opts[:mention].nickname}", - else: "some status" + defp do_generate_activity(users) do + post = %{ + "status" => "Some status without mention with random user" + } - Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_activity_with_mention(user, users) do + mentions_cnt = Enum.random([2, 3, 4, 5]) + with_user = Enum.random([true, false]) + users = Enum.shuffle(users) + mentions_users = Enum.take(users, mentions_cnt) + mentions_users = if with_user, do: [user | mentions_users], else: mentions_users + + mentions_str = + Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ") + + post = %{ + "status" => mentions_str <> "some status with mentions random users" + } + + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_threads(users) do + thread_length = Enum.random([2, 3, 4, 5]) + actor = Enum.random(users) + + post = %{ + "status" => "Start of the thread" + } + + {:ok, activity} = CommonAPI.post(actor, post) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{actor.nickname} reply to thread", + "in_reply_to_status_id" => activity.id + } + + CommonAPI.post(user, post) + end) end def generate_dms(user, users, opts) do @@ -91,22 +170,21 @@ defmodule Pleroma.LoadTesting.Generator do "visibility" => "direct" } - Pleroma.Web.CommonAPI.post(Enum.random(users), post) + CommonAPI.post(Enum.random(users), post) end def generate_long_thread(user, users, opts) do - IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + IO.puts("Starting generating long thread with #{opts[:thread_length]} replies") {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") {:ok, activity} end defp do_generate_long_thread(user, users, opts) do - {:ok, %{id: id} = activity} = - Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"}) Task.async_stream( - 1..opts[:long_thread_length], + 1..opts[:thread_length], fn _ -> do_generate_thread(users, id) end, max_concurrency: 10, timeout: 30_000 @@ -117,50 +195,63 @@ defmodule Pleroma.LoadTesting.Generator do end defp do_generate_thread(users, activity_id) do - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + CommonAPI.post(Enum.random(users), %{ "status" => "reply to main post", "in_reply_to_status_id" => activity_id }) end - def generate_private_thread(users, opts) do - IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") - {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) - IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + def generate_non_visible_message(user, users) do + IO.puts("Starting generating 1000 non visible posts") + + {time, _} = + :timer.tc(fn -> + do_generate_non_visible_posts(user, users) + end) + + IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n") end - defp do_generate_non_visible_posts(users, opts) do - [user1, user2] = Enum.take(users, 2) - {:ok, user1} = Pleroma.User.follow(user1, user2) - {:ok, user2} = Pleroma.User.follow(user2, user1) + defp do_generate_non_visible_posts(user, users) do + [not_friend | users] = users - {:ok, activity} = - Pleroma.Web.CommonAPI.post(user1, %{ - "status" => "Some private post", - "visibility" => "private" - }) + make_friends(user, users) - {:ok, activity_public} = - Pleroma.Web.CommonAPI.post(user2, %{ - "status" => "Some public reply", - "in_reply_to_status_id" => activity.id - }) - - Task.async_stream( - 1..opts[:non_visible_posts_max], - fn _ -> do_generate_non_visible_post(users, activity_public) end, + Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end, max_concurrency: 10, timeout: 30_000 ) + |> Stream.run() end - defp do_generate_non_visible_post(users, activity) do - visibility = Enum.random(["private", "public"]) + defp make_friends(_user, []), do: nil - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ - "visibility" => visibility, - "status" => "Some #{visibility} reply", - "in_reply_to_status_id" => activity.id - }) + defp make_friends(user, [friend | users]) do + {:ok, _} = User.follow(user, friend) + {:ok, _} = User.follow(friend, user) + make_friends(user, users) + end + + defp do_generate_non_visible_post(not_friend, users) do + post = %{ + "status" => "some non visible post", + "visibility" => "private" + } + + {:ok, activity} = CommonAPI.post(not_friend, post) + + thread_length = Enum.random([2, 3, 4, 5]) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{not_friend.nickname} reply to non visible post", + "in_reply_to_status_id" => activity.id, + "visibility" => "private" + } + + CommonAPI.post(user, post) + end) end end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex index 338dba323..47b25c65f 100644 --- a/lib/load_testing/helper.ex +++ b/lib/load_testing/helper.ex @@ -2,13 +2,8 @@ defmodule Pleroma.LoadTesting.Helper do defmacro __using__(_) do quote do import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Notification - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub - alias Pleroma.Web.CommonAPI defp to_sec(microseconds), do: microseconds / 1_000_000 end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index d17cf0b3e..83e531abf 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -13,46 +13,37 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - activities with notifications ## Generate data - MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 - MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + MIX_ENV=benchmark mix pleroma.load_testing --users 10000 + MIX_ENV=benchmark mix pleroma.load_testing -u 10000 Options: - `--users NUMBER` - number of users to generate (default: 10000) - - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities] + @aliases [u: :users, d: :dms, t: :thread_length] @switches [ users: :integer, - activities: :integer, dms: :integer, - thread_length: :integer, - non_visible_posts: :integer + thread_length: :integer ] @users_default 20_000 - @activities_default 50_000 - @dms_default 50_000 + @dms_default 20_000 @thread_length_default 2_000 - @non_visible_posts_default 2_000 def run(args) do start_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) - activities_max = Keyword.get(opts, :activities, @activities_default) dms_max = Keyword.get(opts, :dms, @dms_default) - long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) - non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) + thread_length = Keyword.get(opts, :thread_length, @thread_length_default) clean_tables() opts = Keyword.put(opts, :users_max, users_max) - |> Keyword.put(:activities_max, activities_max) |> Keyword.put(:dms_max, dms_max) - |> Keyword.put(:long_thread_length, long_thread_length) - |> Keyword.put(:non_visible_posts_max, non_visible_posts) + |> Keyword.put(:thread_length, thread_length) generate_users(opts) @@ -60,7 +51,9 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Fetching main user...") {time, user} = - :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + :timer.tc(fn -> + Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") @@ -79,25 +72,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Fetching users take #{to_sec(time)} sec.\n") - generate_activities(users, opts) - - generate_activities(users, Keyword.put(opts, :mention, user)) + generate_activities(user, users) generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) - generate_private_thread(users, opts) - - # generate_replies(user, users, activities) - - # activity = Enum.random(activities) - # generate_long_thread(user, users, activity) + generate_non_visible_message(user, users) IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") - IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") - IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") - IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}") + + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}") + + IO.puts( + "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}" + ) fetch_user(user) query_timelines(user) From 252e5db45c5d9d63116e5a24cc35c26f91ec7de4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 16:37:18 +0300 Subject: [PATCH 04/13] docs fixes --- lib/load_testing/fetcher.ex | 9 --------- lib/load_testing/generator.ex | 3 ++- lib/mix/tasks/pleroma/load_testing.ex | 16 +++++++++++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 825f921e6..b92b6e04f 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -2,8 +2,6 @@ defmodule Pleroma.LoadTesting.Fetcher do use Pleroma.LoadTesting.Helper def fetch_user(user) do - IO.puts("=================================") - Benchee.run(%{ "By id" => fn -> Repo.get_by(User, id: user.id) end, "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, @@ -13,8 +11,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_timelines(user) do - IO.puts("\n=================================") - home_timeline_params = %{ "count" => 20, "with_muted" => true, @@ -102,7 +98,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_notifications(user) do - IO.puts("\n=================================") without_muted_params = %{"count" => "20", "with_muted" => "false"} with_muted_params = %{"count" => "20", "with_muted" => "true"} @@ -138,8 +133,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_dms(user) do - IO.puts("\n=================================") - params = %{ "count" => "20", "with_muted" => "true", @@ -187,8 +180,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_long_thread(user, activity) do - IO.puts("\n=================================") - Benchee.run(%{ "Fetch main post" => fn -> Pleroma.Activity.get_by_id_with_object(activity.id) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 6df518aba..7ad68dcc1 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do nickname: "nick#{i}", password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", - info: %{} + info: %{}, + local: Enum.random([true, false]) } user = %{ diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 83e531abf..4fed6bf74 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -5,19 +5,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do import Pleroma.LoadTesting.Generator import Pleroma.LoadTesting.Fetcher - # tODO: remove autovacuum worker until generation is not ended @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - users - activities with notifications + - direct messages + - long thread + - non visible posts ## Generate data - MIX_ENV=benchmark mix pleroma.load_testing --users 10000 - MIX_ENV=benchmark mix pleroma.load_testing -u 10000 + MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000 + MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000 Options: - - `--users NUMBER` - number of users to generate (default: 10000) + - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u` + - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d` + - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t` """ @aliases [u: :users, d: :dms, t: :thread_length] @@ -32,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do def run(args) do start_pleroma() + Pleroma.Config.put([:instance, :skip_thread_containment], true) {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) @@ -95,11 +100,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do query_notifications(user) query_dms(user) query_long_thread(user, activity) + Pleroma.Config.put([:instance, :skip_thread_containment], false) query_timelines(user) end defp clean_tables do - IO.puts("\n\nDeleting old data...\n") + IO.puts("Deleting old data...\n") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") From b3f6f6a4091c048813e367f6a9414eba6fce5109 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 20:14:02 +0300 Subject: [PATCH 05/13] generating remote users --- lib/load_testing/fetcher.ex | 2 +- lib/load_testing/generator.ex | 34 ++++++++++++++++++++++++++-------- lib/pleroma/user.ex | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index b92b6e04f..0ff2f28c0 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -215,7 +215,7 @@ defmodule Pleroma.LoadTesting.Fetcher do for: user }) end, - "Render context ancestors" => fn -> + "Render context" => fn -> Pleroma.Web.MastodonAPI.StatusView.render( "index.json", for: user, diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7ad68dcc1..61e3d8686 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -22,6 +22,8 @@ defmodule Pleroma.LoadTesting.Generator do end defp generate_user_data(i) do + remote = Enum.random([true, false]) + user = %User{ name: "Test テスト User #{i}", email: "user#{i}@example.com", @@ -29,16 +31,32 @@ defmodule Pleroma.LoadTesting.Generator do password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", info: %{}, - local: Enum.random([true, false]) + local: remote } - user = %{ - user - | ap_id: User.ap_id(user), - follower_address: User.ap_followers(user), - following_address: User.ap_following(user), - following: [User.ap_id(user)] - } + user_urls = + if remote do + base_url = + Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"]) + + ap_id = "#{base_url}/users/#{user.nickname}" + + %{ + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + following: [ap_id] + } + else + %{ + ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + end + + user = Map.merge(user, user_urls) Repo.insert!(user) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2a..37e59a651 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -429,7 +429,9 @@ defmodule Pleroma.User do {:error, "Could not follow user: #{followed.nickname} blocked you."} true -> - if !followed.local && follower.local && !ap_enabled?(followed) do + benchmark? = Pleroma.Config.get([:env]) == :benchmark + + if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do Websub.subscribe(follower, followed) end From 924d7e6aa60095e27c15b4a5f473a192ab2b42ef Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 12:59:36 +0300 Subject: [PATCH 06/13] generating remote activities --- lib/load_testing/generator.ex | 78 ++++++++++++++++++++++++++++++++++- lib/mix/tasks/pleroma/load_testing.ex | 29 +++++++++++-- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 61e3d8686..5c5a5c122 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do name: "Test テスト User #{i}", email: "user#{i}@example.com", nickname: "nick#{i}", - password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + password_hash: + "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg", bio: "Tester Number #{i}", info: %{}, local: remote @@ -165,6 +166,81 @@ defmodule Pleroma.LoadTesting.Generator do end) end + def generate_remote_activities(user, users) do + do_generate_remote_activities(user, users) + end + + defp do_generate_remote_activities(user, users) do + IO.puts("Starting generating 10000 remote activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn i -> + do_generate_remote_activity(i, user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_remote_activity(i, user, users) do + actor = Enum.random(users) + %{host: host} = URI.parse(actor.ap_id) + date = Date.utc_today() + datetime = DateTime.utc_now() + + map = %{ + "actor" => actor.ap_id, + "cc" => [actor.follower_address, user.ap_id], + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "id" => actor.ap_id <> "/statuses/#{i}/activity", + "object" => %{ + "actor" => actor.ap_id, + "atomUri" => actor.ap_id <> "/statuses/#{i}", + "attachment" => [], + "attributedTo" => actor.ap_id, + "bcc" => [], + "bto" => [], + "cc" => [actor.follower_address, user.ap_id], + "content" => + "

+ user.ap_id <> + "\" class=\"u-url mention\">@" <> user.nickname <> "

", + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "conversation" => + "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "emoji" => %{}, + "id" => actor.ap_id <> "/statuses/#{i}", + "inReplyTo" => nil, + "inReplyToAtomUri" => nil, + "published" => datetime, + "sensitive" => true, + "summary" => "cw", + "tag" => [ + %{ + "href" => user.ap_id, + "name" => "@#{user.nickname}@#{host}", + "type" => "Mention" + } + ], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note", + "url" => "http://#{host}/@#{actor.nickname}/#{i}" + }, + "published" => datetime, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create" + } + + Pleroma.Web.ActivityPub.ActivityPub.insert(map, false) + end + def generate_dms(user, users, opts) do IO.puts("Starting generating #{opts[:dms_max]} DMs") {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 4fed6bf74..a7057395d 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -53,32 +53,53 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do generate_users(opts) # main user for queries - IO.puts("Fetching main user...") + IO.puts("Fetching local main user...") {time, user} = :timer.tc(fn -> - Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + Repo.one( + from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1) + ) end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") - IO.puts("Fetching users...") + IO.puts("Fetching local users...") {time, users} = :timer.tc(fn -> Repo.all( from(u in User, where: u.id != ^user.id, + where: u.local == true, order_by: fragment("RANDOM()"), limit: 10 ) ) end) - IO.puts("Fetching users take #{to_sec(time)} sec.\n") + IO.puts("Fetching local users take #{to_sec(time)} sec.\n") + + IO.puts("Fetching remote users...") + + {time, remote_users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + where: u.local == false, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching remote users take #{to_sec(time)} sec.\n") generate_activities(user, users) + generate_remote_activities(user, remote_users) + generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) From 1d285e6fada841b6959b6516aca1c84d1a2c9b93 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 14:02:27 +0300 Subject: [PATCH 07/13] moving to separate dir --- {lib => benchmarks}/load_testing/fetcher.ex | 0 {lib => benchmarks}/load_testing/generator.ex | 0 {lib => benchmarks}/load_testing/helper.ex | 0 {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex | 4 ++-- mix.exs | 1 + 5 files changed, 3 insertions(+), 2 deletions(-) rename {lib => benchmarks}/load_testing/fetcher.ex (100%) rename {lib => benchmarks}/load_testing/generator.ex (100%) rename {lib => benchmarks}/load_testing/helper.ex (100%) rename {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex (98%) diff --git a/lib/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex similarity index 100% rename from lib/load_testing/fetcher.ex rename to benchmarks/load_testing/fetcher.ex diff --git a/lib/load_testing/generator.ex b/benchmarks/load_testing/generator.ex similarity index 100% rename from lib/load_testing/generator.ex rename to benchmarks/load_testing/generator.ex diff --git a/lib/load_testing/helper.ex b/benchmarks/load_testing/helper.ex similarity index 100% rename from lib/load_testing/helper.ex rename to benchmarks/load_testing/helper.ex diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex similarity index 98% rename from lib/mix/tasks/pleroma/load_testing.ex rename to benchmarks/mix/tasks/pleroma/load_testing.ex index a7057395d..7b2293acc 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -8,8 +8,8 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - - users - - activities with notifications + - local/remote users + - local/remote activities with notifications - direct messages - long thread - non visible posts diff --git a/mix.exs b/mix.exs index 3170d6f2d..b0f2c4afb 100644 --- a/mix.exs +++ b/mix.exs @@ -69,6 +69,7 @@ defmodule Pleroma.Mixfile do end # Specifies which paths to compile per environment. + defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"] defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] From 02f8e2a8ab65c3e8497bab4576ce4e75f8df3217 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 14:24:54 +0200 Subject: [PATCH 08/13] Gitlab: Run benchmark in CI. --- .gitlab-ci.yml | 14 ++++++++++++++ config/benchmark.exs | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0c540b16..09684df02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ cache: stages: - build - test + - benchmark - deploy - release @@ -44,6 +45,19 @@ docs-build: paths: - priv/static/doc +benchmark: + stage: benchmark + variables: + MIX_ENV: benchmark + services: + - name: lainsoykaf/postgres-with-rum + alias: postgres + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - mix pleroma.benchmark unit-testing: stage: test diff --git a/config/benchmark.exs b/config/benchmark.exs index b4f5dbdbd..62ba42e28 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -29,7 +29,8 @@ config :pleroma, :instance, email: "admin@example.com", notify_email: "noreply@example.com", skip_thread_containment: false, - federating: false + federating: false, + external_user_synchronization: false config :pleroma, :activitypub, sign_object_fetches: false From 2629493804bb5522903434cef7fe70510725f0c1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:01:06 +0200 Subject: [PATCH 09/13] Benchmark config: Database adjustments. --- config/benchmark.exs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 62ba42e28..289a0e317 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -37,16 +37,11 @@ config :pleroma, :activitypub, sign_object_fetches: false # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, - username: System.get_env("DB_USER") || "postgres", - database: System.get_env("DB_DATABASE") || "pleroma_test", + username: "postgres", + password: "postgres", + database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - # username: "pleroma", - password: - System.get_env("DB_PASS") || - "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", - # password: "", - pool_size: 10, - timeout: 180_000 + pool: Ecto.Adapters.SQL.Sandbox # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From b9b00e6e96b458d9c951ce6505ae903fe47ffa57 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:31:32 +0200 Subject: [PATCH 10/13] Gitlab CI: Run correct benchmark task. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 460c1311f..7c3ca5ee0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ benchmark: - mix deps.get - mix ecto.create - mix ecto.migrate - - mix pleroma.benchmark + - mix pleroma.load_testing unit-testing: stage: test From ecd7ac855bfc6a8e2f95c099b442d9f0f9117e6b Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:42:54 +0200 Subject: [PATCH 11/13] Benchmark config: Don't use the sql sandbox. --- config/benchmark.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 289a0e317..dd99cf5fd 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -41,7 +41,7 @@ config :pleroma, Pleroma.Repo, password: "postgres", database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - pool: Ecto.Adapters.SQL.Sandbox + pool_size: 10 # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From e7bb762ec256bb5dc607797c042aa9432a9aa993 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 15:16:17 +0300 Subject: [PATCH 12/13] don't stream in benchmark env --- lib/pleroma/web/streamer/streamer.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 8cf719277..2fc7ac8cf 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.Streamer do end end - defp handle_should_send(_) do - true - end + defp handle_should_send(:benchmark), do: false + + defp handle_should_send(_), do: true end From 8c0cfed825480ae6338fb1d183dee1440e0ee29c Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 16:26:04 +0300 Subject: [PATCH 13/13] some fixes --- benchmarks/load_testing/fetcher.ex | 2 +- benchmarks/mix/tasks/pleroma/load_testing.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 0ff2f28c0..e378c51e7 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -210,7 +210,7 @@ defmodule Pleroma.LoadTesting.Fetcher do Benchee.run(%{ "Render status" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{ activity: activity, for: user }) diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex index 7b2293acc..4fa3eec49 100644 --- a/benchmarks/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do thread_length: :integer ] @users_default 20_000 - @dms_default 20_000 + @dms_default 1_000 @thread_length_default 2_000 def run(args) do