[#943] Contain search for unauthenticated users See merge request pleroma/pleroma!1220tags/v1.1.4
@@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do | |||
Benchee.run(%{ | |||
"search" => fn -> | |||
Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe") | |||
Pleroma.Activity.search(nil, "cofe") | |||
end | |||
}) | |||
end | |||
@@ -343,4 +343,6 @@ defmodule Pleroma.Activity do | |||
) | |||
) | |||
end | |||
defdelegate search(user, query), to: Pleroma.Activity.Search | |||
end |
@@ -0,0 +1,75 @@ | |||
# Pleroma: A lightweight social networking server | |||
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> | |||
# SPDX-License-Identifier: AGPL-3.0-only | |||
defmodule Pleroma.Activity.Search do | |||
alias Pleroma.Activity | |||
alias Pleroma.Object.Fetcher | |||
alias Pleroma.Repo | |||
alias Pleroma.User | |||
alias Pleroma.Web.ActivityPub.Visibility | |||
import Ecto.Query | |||
def search(user, search_query) do | |||
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin | |||
Activity | |||
|> Activity.with_preloaded_object() | |||
|> Activity.restrict_deactivated_users() | |||
|> restrict_public() | |||
|> query_with(index_type, search_query) | |||
|> maybe_restrict_local(user) | |||
|> Repo.all() | |||
|> maybe_fetch(user, search_query) | |||
end | |||
defp restrict_public(q) do | |||
from([a, o] in q, | |||
where: fragment("?->>'type' = 'Create'", a.data), | |||
where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, | |||
limit: 40 | |||
) | |||
end | |||
defp query_with(q, :gin, search_query) do | |||
from([a, o] in q, | |||
where: | |||
fragment( | |||
"to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", | |||
o.data, | |||
^search_query | |||
), | |||
order_by: [desc: :id] | |||
) | |||
end | |||
defp query_with(q, :rum, search_query) do | |||
from([a, o] in q, | |||
where: | |||
fragment( | |||
"? @@ plainto_tsquery('english', ?)", | |||
o.fts_content, | |||
^search_query | |||
), | |||
order_by: [fragment("? <=> now()::date", o.inserted_at)] | |||
) | |||
end | |||
# users can search everything | |||
defp maybe_restrict_local(q, %User{}), do: q | |||
# unauthenticated users can only search local activities | |||
defp maybe_restrict_local(q, _), do: where(q, local: true) | |||
defp maybe_fetch(activities, user, search_query) do | |||
with true <- Regex.match?(~r/https?:/, search_query), | |||
{:ok, object} <- Fetcher.fetch_object_from_id(search_query), | |||
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), | |||
true <- Visibility.visible_for_user?(activity, user) do | |||
activities ++ [activity] | |||
else | |||
_ -> activities | |||
end | |||
end | |||
end |
@@ -735,122 +735,6 @@ defmodule Pleroma.User do | |||
|> Repo.all() | |||
end | |||
def search(query, resolve \\ false, for_user \\ nil) do | |||
# Strip the beginning @ off if there is a query | |||
query = String.trim_leading(query, "@") | |||
if resolve, do: get_or_fetch(query) | |||
{:ok, results} = | |||
Repo.transaction(fn -> | |||
Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", []) | |||
Repo.all(search_query(query, for_user)) | |||
end) | |||
results | |||
end | |||
def search_query(query, for_user) do | |||
fts_subquery = fts_search_subquery(query) | |||
trigram_subquery = trigram_search_subquery(query) | |||
union_query = from(s in trigram_subquery, union_all: ^fts_subquery) | |||
distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id) | |||
from(s in subquery(boost_search_rank_query(distinct_query, for_user)), | |||
order_by: [desc: s.search_rank], | |||
limit: 40 | |||
) | |||
end | |||
defp boost_search_rank_query(query, nil), do: query | |||
defp boost_search_rank_query(query, for_user) do | |||
friends_ids = get_friends_ids(for_user) | |||
followers_ids = get_followers_ids(for_user) | |||
from(u in subquery(query), | |||
select_merge: %{ | |||
search_rank: | |||
fragment( | |||
""" | |||
CASE WHEN (?) THEN (?) * 1.3 | |||
WHEN (?) THEN (?) * 1.2 | |||
WHEN (?) THEN (?) * 1.1 | |||
ELSE (?) END | |||
""", | |||
u.id in ^friends_ids and u.id in ^followers_ids, | |||
u.search_rank, | |||
u.id in ^friends_ids, | |||
u.search_rank, | |||
u.id in ^followers_ids, | |||
u.search_rank, | |||
u.search_rank | |||
) | |||
} | |||
) | |||
end | |||
defp fts_search_subquery(term, query \\ User) do | |||
processed_query = | |||
term | |||
|> String.replace(~r/\W+/, " ") | |||
|> String.trim() | |||
|> String.split() | |||
|> Enum.map(&(&1 <> ":*")) | |||
|> Enum.join(" | ") | |||
from( | |||
u in query, | |||
select_merge: %{ | |||
search_type: ^0, | |||
search_rank: | |||
fragment( | |||
""" | |||
ts_rank_cd( | |||
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || | |||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'), | |||
to_tsquery('simple', ?), | |||
32 | |||
) | |||
""", | |||
u.nickname, | |||
u.name, | |||
^processed_query | |||
) | |||
}, | |||
where: | |||
fragment( | |||
""" | |||
(setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || | |||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?) | |||
""", | |||
u.nickname, | |||
u.name, | |||
^processed_query | |||
) | |||
) | |||
|> restrict_deactivated() | |||
end | |||
defp trigram_search_subquery(term) do | |||
from( | |||
u in User, | |||
select_merge: %{ | |||
# ^1 gives 'Postgrex expected a binary, got 1' for some weird reason | |||
search_type: fragment("?", 1), | |||
search_rank: | |||
fragment( | |||
"similarity(?, trim(? || ' ' || coalesce(?, '')))", | |||
^term, | |||
u.nickname, | |||
u.name | |||
) | |||
}, | |||
where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term) | |||
) | |||
|> restrict_deactivated() | |||
end | |||
def mute(muter, %User{ap_id: ap_id}) do | |||
info_cng = | |||
muter.info | |||
@@ -1449,4 +1333,6 @@ defmodule Pleroma.User do | |||
) | |||
|> Repo.all() | |||
end | |||
defdelegate search(query, opts \\ []), to: User.Search | |||
end |
@@ -0,0 +1,145 @@ | |||
# Pleroma: A lightweight social networking server | |||
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> | |||
# SPDX-License-Identifier: AGPL-3.0-only | |||
defmodule Pleroma.User.Search do | |||
alias Pleroma.Repo | |||
alias Pleroma.User | |||
import Ecto.Query | |||
def search(query, opts \\ []) do | |||
resolve = Keyword.get(opts, :resolve, false) | |||
for_user = Keyword.get(opts, :for_user) | |||
# Strip the beginning @ off if there is a query | |||
query = String.trim_leading(query, "@") | |||
if match?(%User{}, for_user) and resolve, do: User.get_or_fetch(query) | |||
{:ok, results} = | |||
Repo.transaction(fn -> | |||
Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", []) | |||
query | |||
|> search_query(for_user) | |||
|> Repo.all() | |||
end) | |||
results | |||
end | |||
defp search_query(query, for_user) do | |||
query | |||
|> union_query() | |||
|> distinct_query() | |||
|> boost_search_rank_query(for_user) | |||
|> subquery() | |||
|> order_by(desc: :search_rank) | |||
|> limit(20) | |||
|> maybe_restrict_local(for_user) | |||
end | |||
defp union_query(query) do | |||
fts_subquery = fts_search_subquery(query) | |||
trigram_subquery = trigram_search_subquery(query) | |||
from(s in trigram_subquery, union_all: ^fts_subquery) | |||
end | |||
defp distinct_query(q) do | |||
from(s in subquery(q), order_by: s.search_type, distinct: s.id) | |||
end | |||
# unauthenticated users can only search local activities | |||
defp maybe_restrict_local(q, %User{}), do: q | |||
defp maybe_restrict_local(q, _), do: where(q, [u], u.local == true) | |||
defp boost_search_rank_query(query, nil), do: query | |||
defp boost_search_rank_query(query, for_user) do | |||
friends_ids = User.get_friends_ids(for_user) | |||
followers_ids = User.get_followers_ids(for_user) | |||
from(u in subquery(query), | |||
select_merge: %{ | |||
search_rank: | |||
fragment( | |||
""" | |||
CASE WHEN (?) THEN (?) * 1.3 | |||
WHEN (?) THEN (?) * 1.2 | |||
WHEN (?) THEN (?) * 1.1 | |||
ELSE (?) END | |||
""", | |||
u.id in ^friends_ids and u.id in ^followers_ids, | |||
u.search_rank, | |||
u.id in ^friends_ids, | |||
u.search_rank, | |||
u.id in ^followers_ids, | |||
u.search_rank, | |||
u.search_rank | |||
) | |||
} | |||
) | |||
end | |||
defp fts_search_subquery(term, query \\ User) do | |||
processed_query = | |||
term | |||
|> String.replace(~r/\W+/, " ") | |||
|> String.trim() | |||
|> String.split() | |||
|> Enum.map(&(&1 <> ":*")) | |||
|> Enum.join(" | ") | |||
from( | |||
u in query, | |||
select_merge: %{ | |||
search_type: ^0, | |||
search_rank: | |||
fragment( | |||
""" | |||
ts_rank_cd( | |||
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || | |||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'), | |||
to_tsquery('simple', ?), | |||
32 | |||
) | |||
""", | |||
u.nickname, | |||
u.name, | |||
^processed_query | |||
) | |||
}, | |||
where: | |||
fragment( | |||
""" | |||
(setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || | |||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?) | |||
""", | |||
u.nickname, | |||
u.name, | |||
^processed_query | |||
) | |||
) | |||
|> User.restrict_deactivated() | |||
end | |||
defp trigram_search_subquery(term) do | |||
from( | |||
u in User, | |||
select_merge: %{ | |||
# ^1 gives 'Postgrex expected a binary, got 1' for some weird reason | |||
search_type: fragment("?", 1), | |||
search_rank: | |||
fragment( | |||
"similarity(?, trim(? || ' ' || coalesce(?, '')))", | |||
^term, | |||
u.nickname, | |||
u.name | |||
) | |||
}, | |||
where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term) | |||
) | |||
|> User.restrict_deactivated() | |||
end | |||
end |
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do | |||
alias Pleroma.HTTP | |||
alias Pleroma.Notification | |||
alias Pleroma.Object | |||
alias Pleroma.Object.Fetcher | |||
alias Pleroma.Pagination | |||
alias Pleroma.Repo | |||
alias Pleroma.ScheduledActivity | |||
@@ -1125,64 +1124,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do | |||
end | |||
end | |||
def status_search_query_with_gin(q, query) do | |||
from([a, o] in q, | |||
where: | |||
fragment( | |||
"to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", | |||
o.data, | |||
^query | |||
), | |||
order_by: [desc: :id] | |||
) | |||
end | |||
def status_search_query_with_rum(q, query) do | |||
from([a, o] in q, | |||
where: | |||
fragment( | |||
"? @@ plainto_tsquery('english', ?)", | |||
o.fts_content, | |||
^query | |||
), | |||
order_by: [fragment("? <=> now()::date", o.inserted_at)] | |||
) | |||
end | |||
def status_search(user, query) do | |||
fetched = | |||
if Regex.match?(~r/https?:/, query) do | |||
with {:ok, object} <- Fetcher.fetch_object_from_id(query), | |||
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), | |||
true <- Visibility.visible_for_user?(activity, user) do | |||
[activity] | |||
else | |||
_e -> [] | |||
end | |||
end || [] | |||
q = | |||
from([a, o] in Activity.with_preloaded_object(Activity), | |||
where: fragment("?->>'type' = 'Create'", a.data), | |||
where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, | |||
limit: 40 | |||
) | |||
q = | |||
if Pleroma.Config.get([:database, :rum_enabled]) do | |||
status_search_query_with_rum(q, query) | |||
else | |||
status_search_query_with_gin(q, query) | |||
end | |||
Repo.all(q) ++ fetched | |||
end | |||
def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do | |||
accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user) | |||
statuses = status_search(user, query) | |||
statuses = Activity.search(user, query) | |||
tags_path = Web.base_url() <> "/tag/" | |||
tags = | |||
@@ -1205,8 +1149,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do | |||
def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do | |||
accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user) | |||
statuses = status_search(user, query) | |||
statuses = Activity.search(user, query) | |||
tags = | |||
query | |||
@@ -0,0 +1,7 @@ | |||
defmodule Pleroma.Repo.Migrations.AddIndexOnActivitiesLocal do | |||
use Ecto.Migration | |||
def change do | |||
create(index("activities", [:local])) | |||
end | |||
end |
@@ -99,4 +99,44 @@ defmodule Pleroma.ActivityTest do | |||
assert Activity.get_bookmark(queried_activity, user) == bookmark | |||
end | |||
end | |||
describe "search" do | |||
setup do | |||
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) | |||
user = insert(:user) | |||
params = %{ | |||
"@context" => "https://www.w3.org/ns/activitystreams", | |||
"actor" => "http://mastodon.example.org/users/admin", | |||
"type" => "Create", | |||
"id" => "http://mastodon.example.org/users/admin/activities/1", | |||
"object" => %{ | |||
"type" => "Note", | |||
"content" => "find me!", | |||
"id" => "http://mastodon.example.org/users/admin/objects/1", | |||
"attributedTo" => "http://mastodon.example.org/users/admin" | |||
}, | |||
"to" => ["https://www.w3.org/ns/activitystreams#Public"] | |||
} | |||
{:ok, local_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "find me!"}) | |||
{:ok, remote_activity} = Pleroma.Web.Federator.incoming_ap_doc(params) | |||
%{local_activity: local_activity, remote_activity: remote_activity, user: user} | |||
end | |||
test "find local and remote statuses for authenticated users", %{ | |||
local_activity: local_activity, | |||
remote_activity: remote_activity, | |||
user: user | |||
} do | |||
activities = Enum.sort_by(Activity.search(user, "find me"), & &1.id) | |||
assert [^local_activity, ^remote_activity] = activities | |||
end | |||
test "find only local statuses for unauthenticated users", %{local_activity: local_activity} do | |||
assert [^local_activity] = Activity.search(nil, "find me") | |||
end | |||
end | |||
end |
@@ -1055,7 +1055,7 @@ defmodule Pleroma.UserTest do | |||
u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) | |||
u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) | |||
assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple"), & &1.id) | |||
assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) | |||
end | |||
test "finds users, handling misspelled requests" do | |||
@@ -1077,6 +1077,28 @@ defmodule Pleroma.UserTest do | |||
Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] | |||
end | |||
test "find local and remote statuses for authenticated users" do | |||
u1 = insert(:user, %{name: "lain"}) | |||
u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) | |||
u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) | |||
results = | |||
"lain" | |||
|> User.search(for_user: u1) | |||
|> Enum.map(& &1.id) | |||
|> Enum.sort() | |||
assert [u1.id, u2.id, u3.id] == results | |||
end | |||
test "find only local statuses for unauthenticated users" do | |||
%{id: id} = insert(:user, %{name: "lain"}) | |||
insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) | |||
insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) | |||
assert [%{id: ^id}] = User.search("lain") | |||
end | |||
test "finds a user whose name is nil" do | |||
_user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) | |||
user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) | |||
@@ -1097,7 +1119,11 @@ defmodule Pleroma.UserTest do | |||
end | |||
test "works with URIs" do | |||
results = User.search("http://mastodon.example.org/users/admin", resolve: true) | |||
user = insert(:user) | |||
results = | |||
User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user) | |||
result = results |> List.first() | |||
user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin") | |||
@@ -2173,8 +2173,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do | |||
end | |||
test "search fetches remote accounts", %{conn: conn} do | |||
user = insert(:user) | |||
conn = | |||
conn | |||
|> assign(:user, user) | |||
|> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"}) | |||
assert results = json_response(conn, 200) | |||