changelog merging fix

- added patch edpoint for user tags in api v2 namespace
- returned old put update method
- fixed duplication in users search with tags
- suggested renamings and code changes

CHANGELOG.md

@@ -53,6 +53,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 </details>
 - Improved hashtag timeline performance (requires a background migration). 
 
+<details>
+  <summary>API Changes</summary>
+  - **Breaking** Admin API: methods and routes for users tags.
+</details>
+
 ### Added
 
 - Reports now generate notifications for admins and mods.
@@ -88,6 +93,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: `/api/v1/accounts/:id` & `/api/v1/mutes` endpoints accept `with_relationships` parameter and return filled `pleroma.relationship` field.
 - Mastodon API: Endpoint to remove a conversation (`DELETE /api/v1/conversations/:id`).
 - Mastodon API: `expires_in` in the scheduled post `params` field on `/api/v1/statuses` and `/api/v1/scheduled_statuses/:id` endpoints.
+- Admin API: add `GET /api/pleroma/admin/users/tag` - returns a list of users tags.
 </details>
 
 ### Fixed

docs/development/API/admin_api.md

@@ -119,21 +119,33 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad
 }
 ```
 
-## `PUT /api/v1/pleroma/admin/users/tag`
+## `GET /api/v1/pleroma/admin/user_tags`
+
+### List tags
+
+* Params: None
+* Response:
+
+``` json
+["verify", "mrf_tag:media-force-nsfw"]
+
+```
+
+## `PATCH /api/v1/pleroma/admin/users/tag`
 
 ### Tag a list of users
 
-- Params:
-  - `nicknames` (array)
-  - `tags` (array)
+* Params:
+  * `nicknames` (array)
+  * `tags` (array)
 
 ## `DELETE /api/v1/pleroma/admin/users/tag`
 
 ### Untag a list of users
 
-- Params:
-  - `nicknames` (array)
-  - `tags` (array)
+* Params:
+  * `nicknames` (array)
+  * `tags` (array)
 
 ## `GET /api/v1/pleroma/admin/users/:nickname/permission_group`
 

lib/pleroma/tag.ex

@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Tag do
+  use Ecto.Schema
+
+  import Ecto.Query
+
+  alias Pleroma.Repo
+
+  @mrf_tags [
+    "mrf_tag:media-force-nsfw",
+    "mrf_tag:media-strip",
+    "mrf_tag:force-unlisted",
+    "mrf_tag:sandbox",
+    "mrf_tag:disable-remote-subscription",
+    "mrf_tag:disable-any-subscription"
+  ]
+
+  @type t :: %__MODULE__{}
+
+  schema "tags" do
+    field(:name, :string)
+    many_to_many(:users, Pleroma.User, join_through: "users_tags", on_replace: :delete)
+
+    timestamps()
+  end
+
+  @spec upsert_tags(list(String.t())) :: {integer(), nil | [term()]}
+  def upsert_tags(names) do
+    date = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+    tags =
+      names
+      |> normalize_tags()
+      |> Enum.map(&%{name: &1, inserted_at: date, updated_at: date})
+
+    Repo.insert_all("tags", tags, on_conflict: :nothing, conflict_target: :name)
+  end
+
+  @spec list_tags() :: list(String.t())
+  def list_tags do
+    from(u in __MODULE__, select: u.name)
+    |> Repo.all()
+    |> Kernel.++(@mrf_tags)
+    |> Enum.uniq()
+    |> Enum.sort()
+  end
+
+  @spec get_tag_ids([String.t()]) :: [pos_integer()]
+  def get_tag_ids(tag_names) do
+    names = normalize_tags(tag_names)
+
+    from(
+      u in __MODULE__,
+      select: u.id,
+      where: u.name in ^names
+    )
+    |> Repo.all()
+  end
+
+  @spec normalize_tags([String.t()]) :: [String.t()]
+  def normalize_tags(tag_names) do
+    tag_names
+    |> List.wrap()
+    |> Enum.map(&normalize_tag/1)
+  end
+
+  defp normalize_tag(tag_name) do
+    tag_name
+    |> String.downcase()
+    |> String.trim()
+  end
+end

lib/pleroma/user.ex

@@ -101,7 +101,6 @@ defmodule Pleroma.User do
     field(:following_address, :string)
     field(:search_rank, :float, virtual: true)
     field(:search_type, :integer, virtual: true)
-    field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
     field(:last_digest_emailed_at, :naive_datetime)
     field(:banner, :map, default: %{})
@@ -161,6 +160,7 @@ defmodule Pleroma.User do
 
     has_many(:outgoing_relationships, UserRelationship, foreign_key: :source_id)
     has_many(:incoming_relationships, UserRelationship, foreign_key: :target_id)
+    many_to_many(:tags, Pleroma.Tag, join_through: "users_tags", on_replace: :delete)
 
     for {relationship_type,
          [
@@ -1674,43 +1674,51 @@ defmodule Pleroma.User do
   def purge_user_changeset(user) do
     # "Right to be forgotten"
     # https://gdpr.eu/right-to-be-forgotten/
-    change(user, %{
-      bio: "",
-      raw_bio: nil,
-      email: nil,
-      name: nil,
-      password_hash: nil,
-      keys: nil,
-      public_key: nil,
-      avatar: %{},
-      tags: [],
-      last_refreshed_at: nil,
-      last_digest_emailed_at: nil,
-      banner: %{},
-      background: %{},
-      note_count: 0,
-      follower_count: 0,
-      following_count: 0,
-      is_locked: false,
-      is_confirmed: true,
-      password_reset_pending: false,
-      is_approved: true,
-      registration_reason: nil,
-      confirmation_token: nil,
-      domain_blocks: [],
-      is_active: false,
-      ap_enabled: false,
-      is_moderator: false,
-      is_admin: false,
-      mastofe_settings: nil,
-      mascot: nil,
-      emoji: %{},
-      pleroma_settings_store: %{},
-      fields: [],
-      raw_fields: [],
-      is_discoverable: false,
-      also_known_as: []
-    })
+
+    default_user_attrs =
+      Map.take(
+        %__MODULE__{},
+        [
+          :registration_reason,
+          :is_discoverable,
+          :mastofe_settings,
+          :email,
+          :background,
+          :is_approved,
+          :avatar,
+          :password_hash,
+          :public_key,
+          :mascot,
+          :is_confirmed,
+          :is_locked,
+          :ap_enabled,
+          :note_count,
+          :pleroma_settings_store,
+          :follower_count,
+          :bio,
+          :name,
+          :is_admin,
+          :is_moderator,
+          :also_known_as,
+          :keys,
+          :confirmation_token,
+          :banner,
+          :raw_fields,
+          :fields,
+          :password_reset_pending,
+          :domain_blocks,
+          :last_digest_emailed_at,
+          :raw_bio,
+          :last_refreshed_at,
+          :emoji,
+          :following_count
+        ]
+      )
+
+    user
+    |> Repo.preload([:tags])
+    |> change(Map.merge(default_user_attrs, %{is_active: false}))
+    |> put_assoc(:tags, [])
   end
 
   def delete(users) when is_list(users) do
@@ -1998,6 +2006,15 @@ defmodule Pleroma.User do
 
   def parse_bio(_, _), do: ""
 
+  @spec tag_names(t()) :: [String.t()]
+  def tag_names(%__MODULE__{} = user) do
+    {:ok, tags} = Repo.get_assoc(user, :tags)
+    Enum.map(tags, & &1.name)
+  end
+
+  def tag_names(_), do: []
+
+  @spec tag([String.t()] | String.t() | t(), [String.t()] | String.t()) :: {:ok, [t()]} | t()
   def tag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: tag(user_identifier, tags)
@@ -2007,9 +2024,13 @@ defmodule Pleroma.User do
   def tag(nickname, tags) when is_binary(nickname),
     do: tag(get_by_nickname(nickname), tags)
 
-  def tag(%User{} = user, tags),
-    do: update_tags(user, Enum.uniq((user.tags || []) ++ normalize_tags(tags)))
+  def tag(%User{} = user, tags) do
+    tag_names = Pleroma.Tag.normalize_tags(tags)
+    Pleroma.Tag.upsert_tags(tag_names)
+    append_user_tags(user, tag_names)
+  end
 
+  @spec untag([String.t()] | String.t() | t(), [String.t() | String.t()]) :: {:ok, [t()]} | t()
   def untag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: untag(user_identifier, tags)
@@ -2019,22 +2040,40 @@ defmodule Pleroma.User do
   def untag(nickname, tags) when is_binary(nickname),
     do: untag(get_by_nickname(nickname), tags)
 
-  def untag(%User{} = user, tags),
-    do: update_tags(user, (user.tags || []) -- normalize_tags(tags))
+  def untag(%User{} = user, remove_tags) do
+    tag_ids = Pleroma.Tag.get_tag_ids(remove_tags)
+    {:ok, user_id} = FlakeId.Ecto.Type.dump(user.id)
 
-  defp update_tags(%User{} = user, new_tags) do
-    {:ok, updated_user} =
-      user
-      |> change(%{tags: new_tags})
-      |> update_and_set_cache()
+    from(
+      ut in "users_tags",
+      where: ut.user_id == ^user_id,
+      where: ut.tag_id in ^tag_ids
+    )
+    |> Repo.delete_all()
 
-    updated_user
+    preload_tags_and_set_cache(user)
   end
 
-  defp normalize_tags(tags) do
-    [tags]
-    |> List.flatten()
-    |> Enum.map(&String.downcase/1)
+  defp append_user_tags(%User{} = user, new_tags) do
+    {:ok, user_id} = FlakeId.Ecto.Type.dump(user.id)
+
+    tags =
+      new_tags
+      |> Pleroma.Tag.normalize_tags()
+      |> Pleroma.Tag.get_tag_ids()
+      |> Enum.map(&%{user_id: user_id, tag_id: &1})
+
+    Repo.insert_all("users_tags", tags, on_conflict: :nothing)
+    preload_tags_and_set_cache(user)
+  end
+
+  defp preload_tags_and_set_cache(user) do
+    {:ok, updated_user} =
+      user
+      |> Repo.preload([:tags], force: true)
+      |> set_cache()
+
+    updated_user
   end
 
   defp local_nickname_regex do

lib/pleroma/user/query.ex

@@ -109,7 +109,21 @@ defmodule Pleroma.User.Query do
   end
 
   defp compose_query({:tags, tags}, query) when is_list(tags) and length(tags) > 0 do
-    where(query, [u], fragment("? && ?", u.tags, ^tags))
+    from(u in query,
+      where:
+        fragment(
+          """
+          EXISTS (
+            SELECT 1 FROM tags t
+            JOIN users_tags ut ON ut.tag_id = t.id AND ut.user_id = ?
+            WHERE t.name = ANY(?)
+            LIMIT 1
+          )
+          """,
+          u.id,
+          ^tags
+        )
+    )
   end
 
   defp compose_query({:is_admin, bool}, query) do

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -21,9 +21,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
 
   require Pleroma.Constants
 
-  defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
-  defp get_tags(_), do: []
-
   defp process_tag(
          "mrf_tag:media-force-nsfw",
          %{
@@ -136,8 +133,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
   defp process_tag(_, message), do: {:ok, message}
 
   def filter_message(actor, message) do
-    User.get_cached_by_ap_id(actor)
-    |> get_tags()
+    actor
+    |> User.get_cached_by_ap_id()
+    |> User.tag_names()
     |> Enum.reduce({:ok, message}, fn
       tag, {:ok, message} ->
         process_tag(tag, message)

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -35,8 +35,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [
            :get_password_reset,
            :force_password_reset,
-           :tag_users,
-           :untag_users,
            :right_add,
            :right_add_multiple,
            :right_delete,
@@ -138,32 +136,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
-    with {:ok, _} <- User.tag(nicknames, tags) do
-      ModerationLog.insert_log(%{
-        actor: admin,
-        nicknames: nicknames,
-        tags: tags,
-        action: "tag"
-      })
-
-      json_response(conn, :no_content, "")
-    end
-  end
-
-  def untag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
-    with {:ok, _} <- User.untag(nicknames, tags) do
-      ModerationLog.insert_log(%{
-        actor: admin,
-        nicknames: nicknames,
-        tags: tags,
-        action: "untag"
-      })
-
-      json_response(conn, :no_content, "")
-    end
-  end
-
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames

lib/pleroma/web/admin_api/controllers/tag_controller.ex

@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.TagController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.User
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.ApiSpec
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["admin:write:accounts"]} when action in [:update, :append, :delete]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["admin:read:accounts"]} when action == :index
+  )
+
+  plug(ApiSpec.CastAndValidate)
+
+  action_fallback(AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: ApiSpec.Admin.TagOperation
+
+  def index(%{assigns: %{user: _admin}} = conn, _) do
+    tags = Pleroma.Tag.list_tags()
+
+    json(conn, tags)
+  end
+
+  def update(conn, params), do: append(conn, params)
+
+  def append(
+        %{assigns: %{user: admin}, body_params: %{nicknames: nicknames, tags: tags}} = conn,
+        _
+      ) do
+    with {:ok, _} <- User.tag(nicknames, tags) do
+      ModerationLog.insert_log(%{
+        actor: admin,
+        nicknames: nicknames,
+        tags: tags,
+        action: "tag"
+      })
+
+      json_response(conn, :no_content, "")
+    end
+  end
+
+  def delete(
+        %{assigns: %{user: admin}, body_params: %{nicknames: nicknames, tags: tags}} = conn,
+        _
+      ) do
+    with {:ok, _} <- User.untag(nicknames, tags) do
+      ModerationLog.insert_log(%{
+        actor: admin,
+        nicknames: nicknames,
+        tags: tags,
+        action: "untag"
+      })
+
+      json_response(conn, :no_content, "")
+    end
+  end
+end

lib/pleroma/web/admin_api/views/account_view.ex

@@ -76,7 +76,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
       "is_active" => user.is_active,
       "local" => user.local,
       "roles" => roles(user),
-      "tags" => user.tags || [],
+      "tags" => User.tag_names(user),
       "is_confirmed" => user.is_confirmed,
       "is_approved" => user.is_approved,
       "url" => user.uri || user.ap_id,

lib/pleroma/web/api_spec/operations/admin/tag_operation.ex

@@ -0,0 +1,96 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.TagOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Tags"],
+      summary: "List available tags.",
+      operationId: "AdminAPI.TagController.index",
+      parameters: admin_api_params(),
+      responses: %{
+        200 =>
+          Operation.response("Array of tags", "application/json", %Schema{
+            type: :array,
+            items: %Schema{type: :string}
+          })
+      },
+      security: [%{"oAuth" => ["read:accounts"]}]
+    }
+  end
+
+  def update_operation do
+    %{
+      append_op()
+      | description:
+          "Deprecated. Using [/api/v2/pleroma/admin/users/tags](#operation/AdminAPI.TagController.append) instead is recommended.",
+        operationId: "AdminAPI.TagController.update"
+    }
+  end
+
+  def append_operation, do: append_op()
+
+  defp append_op do
+    %Operation{
+      tags: ["Admin", "Tags"],
+      summary: "Adds tags to users.",
+      operationId: "AdminAPI.TagController.append",
+      parameters: admin_api_params(),
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            properties: %{
+              nicknames: %Schema{type: :array, items: %Schema{type: :string}},
+              tags: %Schema{type: :array, items: %Schema{type: :string}}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        204 => no_content_response(),
+        400 => Operation.response("Bad request", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:accounts"]}]
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "Tags"],
+      summary: "Remove tags from users.",
+      operationId: "AdminAPI.TagController.delete",
+      parameters: admin_api_params(),
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            properties: %{
+              nicknames: %Schema{type: :array, items: %Schema{type: :string}},
+              tags: %Schema{type: :array, items: %Schema{type: :string}}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        204 => no_content_response(),
+        400 => Operation.response("Bad request", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:accounts"]}]
+    }
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -269,7 +269,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
         ap_id: user.ap_id,
         also_known_as: user.also_known_as,
         is_confirmed: user.is_confirmed,
-        tags: user.tags,
+        tags: User.tag_names(user),
         hide_followers_count: user.hide_followers_count,
         hide_follows_count: user.hide_follows_count,
         hide_followers: user.hide_followers,

lib/pleroma/web/router.ex

@@ -155,12 +155,19 @@ defmodule Pleroma.Web.Router do
     post("/uploader_callback/:upload_path", UploaderController, :callback)
   end
 
+  scope "/api/v2/pleroma/admin", Pleroma.Web.AdminAPI do
+    pipe_through(:admin_api)
+    patch("/users/tags", TagController, :append)
+  end
+
   scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
 
     put("/users/disable_mfa", AdminAPIController, :disable_mfa)
-    put("/users/tag", AdminAPIController, :tag_users)
-    delete("/users/tag", AdminAPIController, :untag_users)
+
+    get("/user_tags", TagController, :index)
+    put("/users/tags", TagController, :update)
+    delete("/users/tags", TagController, :delete)
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)

priv/repo/migrations/20201012093959_create_tags.exs

@@ -0,0 +1,32 @@
+defmodule Pleroma.Repo.Migrations.CreateTags do
+  use Ecto.Migration
+
+  def up do
+    create_if_not_exists table(:tags) do
+      add(:name, :string, null: false)
+      timestamps()
+    end
+
+    create_if_not_exists(unique_index(:tags, :name))
+    flush()
+
+    Ecto.Adapters.SQL.query(
+      Pleroma.Repo,
+      collect_user_tags_query(),
+      [],
+      timeout: :infinity
+    )
+  end
+
+  def down do
+    drop_if_exists(table(:tags))
+  end
+
+  defp collect_user_tags_query do
+    """
+    INSERT INTO tags(name, inserted_at, updated_at)
+    SELECT DISTINCT TRIM(unnest(tags)), now(), now() from users
+    ON CONFLICT DO NOTHING
+    """
+  end
+end

priv/repo/migrations/20201014064744_create_user_tag.exs

@@ -0,0 +1,13 @@
+defmodule Pleroma.Repo.Migrations.CreateUserTag do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:users_tags, primary_key: false) do
+      add(:tag_id, references(:tags, on_delete: :delete_all))
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+    end
+
+    create_if_not_exists(index(:users_tags, [:tag_id]))
+    create_if_not_exists(unique_index(:users_tags, [:user_id, :tag_id]))
+  end
+end

priv/repo/migrations/20201021180413_import_legacy_tags_to_users_tags.exs

@@ -0,0 +1,65 @@
+defmodule Pleroma.Repo.Migrations.ImportLegacyTagsToUsersTags do
+  use Ecto.Migration
+
+  def up do
+    Ecto.Adapters.SQL.query(
+      Pleroma.Repo,
+      import_user_tags(),
+      [],
+      timeout: :infinity
+    )
+
+    alter table(:users) do
+      remove_if_exists(:tags, {:array, :string})
+    end
+
+    drop_if_exists(index(:users, [:tags]))
+  end
+
+  def down do
+    alter table(:users) do
+      add_if_not_exists(:tags, {:array, :string}, default: [], null: false)
+    end
+
+    create_if_not_exists(index(:users, [:tags], using: :gin))
+
+    flush()
+
+    Ecto.Adapters.SQL.query(
+      Pleroma.Repo,
+      restore_tags_column(),
+      [],
+      timeout: :infinity
+    )
+  end
+
+  defp import_user_tags do
+    """
+    INSERT INTO users_tags(user_id, tag_id)
+    SELECT user_tags.user_id, tags.id
+     FROM (
+     SELECT  DISTINCT TRIM(unnest(tags)) as "tag", id as "user_id"
+       FROM users ) as "user_tags"
+     INNER JOIN tags as tags on tags.name = user_tags."tag"
+     ON CONFLICT DO NOTHING
+    """
+  end
+
+  defp restore_tags_column do
+    """
+    UPDATE
+      users
+    SET
+      tags = tags_query.tags_array,
+      updated_at = now()
+    FROM (
+      SELECT user_id, array_agg(tags.name) as tags_array
+      FROM users_tags
+        INNER JOIN users ON users.id = user_id
+        INNER JOIN tags ON tags.id = tag_id
+      GROUP BY user_id
+    ) as tags_query
+    WHERE tags_query.user_id = users.id
+    """
+  end
+end

test/mix/tasks/pleroma/user_test.exs

@@ -548,7 +548,7 @@ defmodule Mix.Tasks.Pleroma.UserTest do
       :ok = Mix.Tasks.Pleroma.User.run(["tag", user.nickname, "pleroma"])
 
       user = User.get_cached_by_nickname(user.nickname)
-      assert "pleroma" in user.tags
+      assert "pleroma" in Enum.map(user.tags, & &1.name)
     end
 
     test "it prints an error message when user is not exist" do
@@ -561,8 +561,8 @@ defmodule Mix.Tasks.Pleroma.UserTest do
 
   describe "untagging" do
     test "it deletes tags from a user" do
-      user = insert(:user, tags: ["pleroma"])
-      assert "pleroma" in user.tags
+      user = insert(:user, tags: [build(:tag, name: "pleroma")])
+      assert "pleroma" in Enum.map(user.tags, & &1.name)
 
       :ok = Mix.Tasks.Pleroma.User.run(["untag", user.nickname, "pleroma"])
 

test/pleroma/repo/migrations/fix_legacy_tags_test.exs

@@ -1,28 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Repo.Migrations.FixLegacyTagsTest do
-  alias Pleroma.User
-  use Pleroma.DataCase, async: true
-  import Pleroma.Factory
-  import Pleroma.Tests.Helpers
-
-  setup_all do: require_migration("20200802170532_fix_legacy_tags")
-
-  test "change/0 converts legacy user tags into correct values", %{migration: migration} do
-    user = insert(:user, tags: ["force_nsfw", "force_unlisted", "verified"])
-    user2 = insert(:user)
-
-    assert :ok == migration.change()
-
-    fixed_user = User.get_by_id(user.id)
-    fixed_user2 = User.get_by_id(user2.id)
-
-    assert fixed_user.tags == ["mrf_tag:media-force-nsfw", "mrf_tag:force-unlisted", "verified"]
-    assert fixed_user2.tags == []
-
-    # user2 should not have been updated
-    assert fixed_user2.updated_at == fixed_user2.inserted_at
-  end
-end

test/pleroma/tag_test.exs

@@ -0,0 +1,70 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.TagTest do
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  alias Pleroma.Tag
+
+  describe "upsert_tags/1" do
+    test "create new normalize tags" do
+      Tag.upsert_tags(["  verify   \n", "bot", "unconfirmed   "])
+
+      assert tags = Pleroma.Repo.all(Tag)
+      assert Enum.sort(Enum.map(tags, & &1.name)) == ["bot", "unconfirmed", "verify"]
+    end
+
+    test "do nothing when tag exists" do
+      insert(:tag, name: "verify")
+
+      Tag.upsert_tags(["  verify   \n", "bot", "unconfirmed   "])
+
+      assert tags = Pleroma.Repo.all(Tag)
+      assert Enum.sort(Enum.map(tags, & &1.name)) == ["bot", "unconfirmed", "verify"]
+    end
+  end
+
+  describe "get_tag_ids/1" do
+    test "returns tags by name" do
+      verify_tag = insert(:tag, name: "verify")
+      bot_tag = insert(:tag, name: "bot")
+      unconfirmed_tag = insert(:tag, name: "unconfirmed")
+
+      tag_ids = Tag.get_tag_ids(["bot", "verify"])
+      assert verify_tag.id in tag_ids
+      assert bot_tag.id in tag_ids
+      refute unconfirmed_tag.id in tag_ids
+    end
+  end
+
+  describe "list_tags/0" do
+    test "returns all users tags + mrf tags" do
+      insert(:tag, name: "verify")
+      insert(:tag, name: "bot")
+      insert(:tag, name: "unconfirmed")
+      insert(:tag, name: "mrf_tag:media-strip")
+
+      assert Enum.sort(Tag.list_tags()) == [
+               "bot",
+               "mrf_tag:disable-any-subscription",
+               "mrf_tag:disable-remote-subscription",
+               "mrf_tag:force-unlisted",
+               "mrf_tag:media-force-nsfw",
+               "mrf_tag:media-strip",
+               "mrf_tag:sandbox",
+               "unconfirmed",
+               "verify"
+             ]
+    end
+  end
+
+  describe "normalize_tags/1" do
+    test "returns normalize tags" do
+      assert ["verify", "bot"] == Tag.normalize_tags(["  verify \n", "\n  bot  "])
+      assert ["verify"] == Tag.normalize_tags("  verify \n")
+    end
+  end
+end

test/pleroma/user_test.exs

@@ -133,21 +133,39 @@ defmodule Pleroma.UserTest do
 
   describe "when tags are nil" do
     test "tagging a user" do
-      user = insert(:user, %{tags: nil})
-      user = User.tag(user, ["cool", "dude"])
+      insert(:tag, name: "cool")
+      insert(:tag, name: "dude")
+      user = insert(:user, %{tags: []})
 
-      assert "cool" in user.tags
-      assert "dude" in user.tags
+      user = User.tag(user, ["cool", "dude"])
+      user_tags = Enum.map(user.tags, & &1.name)
+
+      assert "cool" in user_tags
+      assert "dude" in user_tags
     end
 
     test "untagging a user" do
-      user = insert(:user, %{tags: nil})
+      user = insert(:user, %{tags: []})
       user = User.untag(user, ["cool", "dude"])
 
       assert user.tags == []
     end
   end
 
+  describe "tag_names/1" do
+    test "returns tag names of user" do
+      user =
+        insert(:user, %{
+          tags: [
+            build(:tag, name: "verify"),
+            build(:tag, name: "spam")
+          ]
+        })
+
+      assert User.tag_names(user) == ["verify", "spam"]
+    end
+  end
+
   test "ap_id returns the activity pub id for the user" do
     user = UserBuilder.build()
 
@@ -1614,7 +1632,7 @@ defmodule Pleroma.UserTest do
         keys: "RSA begin buplic key",
         public_key: "--PRIVATE KEYE--",
         avatar: %{"a" => "b"},
-        tags: ["qqqqq"],
+        tags: [build(:tag, name: "verify")],
         banner: %{"a" => "b"},
         background: %{"a" => "b"},
         note_count: 9,
@@ -1641,9 +1659,16 @@ defmodule Pleroma.UserTest do
         also_known_as: ["https://lol.olo/users/loll"]
       })
 
+    assert Enum.map(user.tags, & &1.name) == ["verify"]
+    assert Repo.aggregate(from(ut in "users_tags"), :count, :user_id) == 1
     {:ok, job} = User.delete(user)
     {:ok, _} = ObanHelpers.perform(job)
-    user = User.get_by_id(user.id)
+
+    user =
+      User.get_by_id(user.id)
+      |> Repo.preload([:tags])
+
+    assert Repo.aggregate(from(ut in "users_tags"), :count, :user_id) == 0
 
     assert %User{
              bio: "",

test/pleroma/web/activity_pub/mrf/tag_policy_test.exs

@@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:disable-any-subscription" do
     test "rejects message" do
-      actor = insert(:user, tags: ["mrf_tag:disable-any-subscription"])
+      actor = insert(:user, tags: [build(:tag, name: "mrf_tag:disable-any-subscription")])
       message = %{"object" => actor.ap_id, "type" => "Follow", "actor" => actor.ap_id}
       assert {:reject, _} = TagPolicy.filter(message)
     end
@@ -19,15 +19,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:disable-remote-subscription" do
     test "rejects non-local follow requests" do
-      actor = insert(:user, tags: ["mrf_tag:disable-remote-subscription"])
-      follower = insert(:user, tags: ["mrf_tag:disable-remote-subscription"], local: false)
+      tag = insert(:tag, name: "mrf_tag:disable-remote-subscription")
+      actor = insert(:user, tags: [tag])
+      follower = insert(:user, tags: [tag], local: false)
       message = %{"object" => actor.ap_id, "type" => "Follow", "actor" => follower.ap_id}
       assert {:reject, _} = TagPolicy.filter(message)
     end
 
     test "allows non-local follow requests" do
-      actor = insert(:user, tags: ["mrf_tag:disable-remote-subscription"])
-      follower = insert(:user, tags: ["mrf_tag:disable-remote-subscription"], local: true)
+      tag = insert(:tag, name: "mrf_tag:disable-remote-subscription")
+      actor = insert(:user, tags: [tag])
+      follower = insert(:user, tags: [tag], local: true)
       message = %{"object" => actor.ap_id, "type" => "Follow", "actor" => follower.ap_id}
       assert {:ok, _message} = TagPolicy.filter(message)
     end
@@ -35,7 +37,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:sandbox" do
     test "removes from public timelines" do
-      actor = insert(:user, tags: ["mrf_tag:sandbox"])
+      actor = insert(:user, tags: [build(:tag, name: "mrf_tag:sandbox")])
 
       message = %{
         "actor" => actor.ap_id,
@@ -59,7 +61,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:force-unlisted" do
     test "removes from the federated timeline" do
-      actor = insert(:user, tags: ["mrf_tag:force-unlisted"])
+      actor = insert(:user, tags: [build(:tag, name: "mrf_tag:force-unlisted")])
 
       message = %{
         "actor" => actor.ap_id,
@@ -83,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:media-strip" do
     test "removes attachments" do
-      actor = insert(:user, tags: ["mrf_tag:media-strip"])
+      actor = insert(:user, tags: [build(:tag, name: "mrf_tag:media-strip")])
 
       message = %{
         "actor" => actor.ap_id,
@@ -103,7 +105,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicyTest do
 
   describe "mrf_tag:media-force-nsfw" do
     test "Mark as sensitive on presence of attachments" do
-      actor = insert(:user, tags: ["mrf_tag:media-force-nsfw"])
+      actor = insert(:user, tags: [build(:tag, name: "mrf_tag:media-force-nsfw")])
 
       message = %{
         "actor" => actor.ap_id,

test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs

@@ -90,98 +90,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     end
   end
 
-  describe "PUT /api/pleroma/admin/users/tag" do
-    setup %{conn: conn} do
-      user1 = insert(:user, %{tags: ["x"]})
-      user2 = insert(:user, %{tags: ["y"]})
-      user3 = insert(:user, %{tags: ["unchanged"]})
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> put(
-          "/api/pleroma/admin/users/tag?nicknames[]=#{user1.nickname}&nicknames[]=" <>
-            "#{user2.nickname}&tags[]=foo&tags[]=bar"
-        )
-
-      %{conn: conn, user1: user1, user2: user2, user3: user3}
-    end
-
-    test "it appends specified tags to users with specified nicknames", %{
-      conn: conn,
-      admin: admin,
-      user1: user1,
-      user2: user2
-    } do
-      assert empty_json_response(conn)
-      assert User.get_cached_by_id(user1.id).tags == ["x", "foo", "bar"]
-      assert User.get_cached_by_id(user2.id).tags == ["y", "foo", "bar"]
-
-      log_entry = Repo.one(ModerationLog)
-
-      users =
-        [user1.nickname, user2.nickname]
-        |> Enum.map(&"@#{&1}")
-        |> Enum.join(", ")
-
-      tags = ["foo", "bar"] |> Enum.join(", ")
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} added tags: #{tags} to users: #{users}"
-    end
-
-    test "it does not modify tags of not specified users", %{conn: conn, user3: user3} do
-      assert empty_json_response(conn)
-      assert User.get_cached_by_id(user3.id).tags == ["unchanged"]
-    end
-  end
-
-  describe "DELETE /api/pleroma/admin/users/tag" do
-    setup %{conn: conn} do
-      user1 = insert(:user, %{tags: ["x"]})
-      user2 = insert(:user, %{tags: ["y", "z"]})
-      user3 = insert(:user, %{tags: ["unchanged"]})
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> delete(
-          "/api/pleroma/admin/users/tag?nicknames[]=#{user1.nickname}&nicknames[]=" <>
-            "#{user2.nickname}&tags[]=x&tags[]=z"
-        )
-
-      %{conn: conn, user1: user1, user2: user2, user3: user3}
-    end
-
-    test "it removes specified tags from users with specified nicknames", %{
-      conn: conn,
-      admin: admin,
-      user1: user1,
-      user2: user2
-    } do
-      assert empty_json_response(conn)
-      assert User.get_cached_by_id(user1.id).tags == []
-      assert User.get_cached_by_id(user2.id).tags == ["y"]
-
-      log_entry = Repo.one(ModerationLog)
-
-      users =
-        [user1.nickname, user2.nickname]
-        |> Enum.map(&"@#{&1}")
-        |> Enum.join(", ")
-
-      tags = ["x", "z"] |> Enum.join(", ")
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} removed tags: #{tags} from users: #{users}"
-    end
-
-    test "it does not modify tags of not specified users", %{conn: conn, user3: user3} do
-      assert empty_json_response(conn)
-      assert User.get_cached_by_id(user3.id).tags == ["unchanged"]
-    end
-  end
-
   describe "/api/pleroma/admin/users/:nickname/permission_group" do
     test "GET is giving user_info", %{admin: admin, conn: conn} do
       conn =

test/pleroma/web/admin_api/controllers/tag_controller_test.exs

@@ -0,0 +1,190 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.TagControllerTest do
+  use Pleroma.Web.ConnCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Pleroma.Factory
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "GET /api/pleroma/admin/users/tags" do
+    test "it returns user tags and mrf policy tags", %{conn: conn} do
+      insert(:tag, name: "x")
+      insert(:tag, name: "y")
+      insert(:tag, name: "unchanged")
+
+      response =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> get("/api/pleroma/admin/user_tags")
+        |> json_response_and_validate_schema(200)
+
+      assert [
+               "mrf_tag:disable-any-subscription",
+               "mrf_tag:disable-remote-subscription",
+               "mrf_tag:force-unlisted",
+               "mrf_tag:media-force-nsfw",
+               "mrf_tag:media-strip",
+               "mrf_tag:sandbox",
+               "unchanged",
+               "x",
+               "y"
+             ] == response
+    end
+  end
+
+  describe "PUT /api/pleroma/admin/users/tags" do
+    setup %{conn: conn} do
+      user1 = insert(:user, %{tags: [build(:tag, name: "x")]})
+      user2 = insert(:user, %{tags: [build(:tag, name: "y")]})
+      user3 = insert(:user, %{tags: [build(:tag, name: "unchanged")]})
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> put("/api/pleroma/admin/users/tags", %{
+               nicknames: [user1.nickname, user2.nickname],
+               tags: ["foo", "bar"]
+             })
+             |> json_response_and_validate_schema(204)
+
+      %{user1: user1, user2: user2, user3: user3}
+    end
+
+    test "it appends specified tags to users with specified nicknames", %{
+      admin: admin,
+      user1: user1,
+      user2: user2
+    } do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user1.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["x", "foo", "bar"]
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user2.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["y", "foo", "bar"]
+
+      log_entry = Repo.one(ModerationLog)
+
+      users =
+        [user1.nickname, user2.nickname]
+        |> Enum.map(&"@#{&1}")
+        |> Enum.join(", ")
+
+      tags = ["foo", "bar"] |> Enum.join(", ")
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} added tags: #{tags} to users: #{users}"
+    end
+
+    test "it does not modify tags of not specified users", %{user3: user3} do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user3.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["unchanged"]
+    end
+  end
+
+  describe "PATCH /api/v2/pleroma/admin/users/tags" do
+    setup %{conn: conn} do
+      user1 = insert(:user, %{tags: [build(:tag, name: "x")]})
+      user2 = insert(:user, %{tags: [build(:tag, name: "y")]})
+      user3 = insert(:user, %{tags: [build(:tag, name: "unchanged")]})
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> patch("/api/v2/pleroma/admin/users/tags", %{
+               nicknames: [user1.nickname, user2.nickname],
+               tags: ["foo", "bar"]
+             })
+             |> json_response_and_validate_schema(204)
+
+      %{user1: user1, user2: user2, user3: user3}
+    end
+
+    test "it appends specified tags to users with specified nicknames", %{
+      admin: admin,
+      user1: user1,
+      user2: user2
+    } do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user1.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["x", "foo", "bar"]
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user2.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["y", "foo", "bar"]
+
+      log_entry = Repo.one(ModerationLog)
+
+      users =
+        [user1.nickname, user2.nickname]
+        |> Enum.map(&"@#{&1}")
+        |> Enum.join(", ")
+
+      tags = ["foo", "bar"] |> Enum.join(", ")
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} added tags: #{tags} to users: #{users}"
+    end
+
+    test "it does not modify tags of not specified users", %{user3: user3} do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user3.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["unchanged"]
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/users/tags" do
+    setup %{conn: conn} do
+      user1 = insert(:user, %{tags: [build(:tag, name: "x")]})
+      user2 = insert(:user, %{tags: [build(:tag, name: "y"), build(:tag, name: "z")]})
+      user3 = insert(:user, %{tags: [build(:tag, name: "unchanged")]})
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> delete(
+               "/api/pleroma/admin/users/tags",
+               %{nicknames: [user1.nickname, user2.nickname], tags: ["x", "z"]}
+             )
+             |> json_response_and_validate_schema(204)
+
+      %{user1: user1, user2: user2, user3: user3}
+    end
+
+    test "it removes specified tags from users with specified nicknames", %{
+      admin: admin,
+      user1: user1,
+      user2: user2
+    } do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user1.id), :tags)
+      assert tags == []
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user2.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["y"]
+
+      log_entry = Repo.one(ModerationLog)
+
+      users =
+        [user1.nickname, user2.nickname]
+        |> Enum.map(&"@#{&1}")
+        |> Enum.join(", ")
+
+      tags = ["x", "z"] |> Enum.join(", ")
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} removed tags: #{tags} from users: #{users}"
+    end
+
+    test "it does not modify tags of not specified users", %{user3: user3} do
+      {:ok, tags} = Repo.get_assoc(User.get_cached_by_id(user3.id), :tags)
+      assert Enum.map(tags, & &1.name) == ["unchanged"]
+    end
+  end
+end

test/pleroma/web/admin_api/controllers/user_controller_test.exs

@@ -379,7 +379,9 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do
 
   describe "GET /api/pleroma/admin/users" do
     test "renders users array for the first page", %{conn: conn, admin: admin} do
-      user = insert(:user, local: false, tags: ["foo", "bar"])
+      user =
+        insert(:user, local: false, tags: [build(:tag, name: "foo"), build(:tag, name: "bar")])
+
       user2 = insert(:user, is_approved: false, registration_reason: "I'm a chill dude")
 
       conn = get(conn, "/api/pleroma/admin/users?page=1")
@@ -745,8 +747,8 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do
     end
 
     test "load users with tags list", %{conn: conn} do
-      user1 = insert(:user, tags: ["first"])
-      user2 = insert(:user, tags: ["second"])
+      user1 = insert(:user, tags: [build(:tag, name: "first")])
+      user2 = insert(:user, tags: [build(:tag, name: "second")])
       insert(:user)
       insert(:user)
 

test/pleroma/web/admin_api/search_test.exs

@@ -130,17 +130,18 @@ defmodule Pleroma.Web.AdminAPI.SearchTest do
     end
 
     test "it returns users with tags" do
-      user1 = insert(:user, tags: ["first"])
-      user2 = insert(:user, tags: ["second"])
+      user1 = insert(:user, tags: [build(:tag, name: "first")])
+      user2 = insert(:user, tags: [build(:tag, name: "second")])
       insert(:user)
       insert(:user)
 
       {:ok, _results, total} = Search.user()
       {:ok, users, count} = Search.user(%{tags: ["first", "second"]})
+
       assert total == 4
       assert count == 2
-      assert user1 in users
-      assert user2 in users
+      assert user1.id in collect_ids(users)
+      assert user2.id in collect_ids(users)
     end
 
     test "it returns users by actor_types" do
@@ -212,5 +213,17 @@ defmodule Pleroma.Web.AdminAPI.SearchTest do
 
       assert total == 2
     end
+
+    test "tags deduplication" do
+      first_tag = insert(:tag, name: "first")
+      second_tag = insert(:tag, name: "second")
+
+      user1 = insert(:user, tags: [first_tag, second_tag])
+      user2 = insert(:user, tags: [first_tag, second_tag])
+
+      {:ok, users, count} = Search.user(%{tags: ["first", "second"], page_size: 2})
+
+      assert {Enum.sort(Enum.map(users, & &1.id)), count} == {Enum.sort([user1.id, user2.id]), 2}
+    end
   end
 end

test/support/factory.ex

@@ -490,4 +490,8 @@ defmodule Pleroma.Factory do
       context: ["home"]
     }
   end
+
+  def tag_factory do
+    %Pleroma.Tag{name: "verify"}
+  end
 end